From 8992ff8e48c2f38aa8284035ec0a631523c1101a Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Wed, 19 Aug 2026 17:44:54 +1000 Subject: [PATCH] Intel x64 ASM: intrinsic output Add another implementation of the assembly code for Intel x64 that uses intrinsics and C code. --- .wolfssl_known_macro_extras | 2 + CMakeLists.txt | 147 +- cmake/functions.cmake | 183 +- configure.ac | 36 + src/include.am | 206 +- wolfcrypt/src/aes_gcm_intrin.c | 45216 +++++++++++++++++ wolfcrypt/src/aes_x86_64_intrin.c | 6652 +++ wolfcrypt/src/aes_xts_intrin.c | 9735 ++++ wolfcrypt/src/chacha_intrin.c | 11594 +++++ wolfcrypt/src/fe_x25519_asm.asm | 520 + wolfcrypt/src/fe_x25519_intrin.c | 21710 +++++++++ wolfcrypt/src/poly1305_intrin.c | 3677 ++ wolfcrypt/src/sha256_asm.asm | 4 + wolfcrypt/src/sha256_x86_64_intrin.c | 23636 +++++++++ wolfcrypt/src/sha3_x86_64_intrin.c | 56837 ++++++++++++++++++++++ wolfcrypt/src/sha512_asm.asm | 6 +- wolfcrypt/src/sha512_x86_64_intrin.c | 10912 +++++ wolfcrypt/src/sp_x86_64_asm.S | 4 +- wolfcrypt/src/sp_x86_64_asm.asm | 4 +- wolfcrypt/src/sp_x86_64_intrin.c | 53134 ++++++++++++++++++++ wolfcrypt/src/wc_falcon_fpr_intrin.c | 512 + wolfcrypt/src/wc_frodokem_intrin.c | 1999 + wolfcrypt/src/wc_mldsa_intrin.c | 64033 +++++++++++++++++++++++++ wolfcrypt/src/wc_mlkem_intrin.c | 39467 +++++++++++++++ 24 files changed, 350195 insertions(+), 31 deletions(-) create mode 100644 wolfcrypt/src/aes_gcm_intrin.c create mode 100644 wolfcrypt/src/aes_x86_64_intrin.c create mode 100644 wolfcrypt/src/aes_xts_intrin.c create mode 100644 wolfcrypt/src/chacha_intrin.c create mode 100644 wolfcrypt/src/fe_x25519_intrin.c create mode 100644 wolfcrypt/src/poly1305_intrin.c create mode 100644 wolfcrypt/src/sha256_x86_64_intrin.c create mode 100644 wolfcrypt/src/sha3_x86_64_intrin.c create mode 100644 wolfcrypt/src/sha512_x86_64_intrin.c create mode 100644 wolfcrypt/src/sp_x86_64_intrin.c create mode 100644 wolfcrypt/src/wc_falcon_fpr_intrin.c create mode 100644 wolfcrypt/src/wc_frodokem_intrin.c create mode 100644 wolfcrypt/src/wc_mldsa_intrin.c create mode 100644 wolfcrypt/src/wc_mlkem_intrin.c diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 5edebd83ef1..67d9601c408 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -1140,6 +1140,7 @@ WOLFSSL_X509_TINY_POLICIES WOLFSSL_X509_TINY_SKI WOLFSSL_X509_TRUSTED_CERTIFICATE_CALLBACK WOLFSSL_X509_VERIFY_ONLY +WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA WOLFSSL_XFREE_NO_NULLNESS_CHECK WOLFSSL_XILINX_CRYPTO_OLD WOLFSSL_XILINX_PATCH @@ -1218,6 +1219,7 @@ __ATOMIC_CONSUME __ATOMIC_RELAXED __AVR_ARCH__ __AVR__ +__AVX2__ __AVX512F__ __BCPLUSPLUS__ __BIG_ENDIAN__ diff --git a/CMakeLists.txt b/CMakeLists.txt index f35291e03d2..1399e8b6b7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2177,8 +2177,7 @@ else() endif() endif() -# TODO: - AES-XTS -# - Web server +# TODO: - Web server # - Web client add_option("WOLFSSL_CMAC" "Enable CMAC (default: disabled)" @@ -2763,10 +2762,128 @@ if("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "x86_64|AMD64") set(WOLFSSL_X86_64_BUILD ON) add_option("WOLFSSL_X86_64_BUILD_ASM" "Build ASM files" "yes" "yes;no") list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_X86_64_BUILD") +elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "^(i[3-6]86|x86)$") + # 32-bit x86. The distinction matters beyond the define: the AES-GCM + # speedup has its OWN assembly there (aes_gcm_x86_asm.S) rather than a + # 32-bit build of the x86-64 one, which is what BUILD_X86_ASM selects + # below - the same split autotools makes with ENABLED_X86_ASM. + set(WOLFSSL_X86_BUILD ON) + list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_X86_BUILD") elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "aarch64|arm64") list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AARCH64_BUILD") endif() +# Intel assembly speedups. cmake/functions.cmake already selects the asm +# sources from this variable; declaring it makes it discoverable and gives it +# a default. USE_INTEL_SPEEDUP is what makes the C dispatch to those sources - +# without it they compile but nothing ever calls them. +add_option("WOLFSSL_INTEL_ASM" + "Enable Intel assembly speedups (default: disabled)" + "no" "yes;no") + +if(WOLFSSL_INTEL_ASM) + list(APPEND WOLFSSL_DEFINITIONS "-DUSE_INTEL_SPEEDUP") +endif() + +# AES-NI. Same shape: functions.cmake already reads WOLFSSL_AESNI to select +# the AES assembly, but WOLFSSL_AESNI has to reach the compiler as well or +# aes.c leaves every AES-NI path out and the assembly it does build is never +# called. --enable-intelasm implies it in the autotools build; match that. +add_option("WOLFSSL_AESNI" + "Enable AES-NI (default: disabled)" + "no" "yes;no") + +if(WOLFSSL_AESNI OR WOLFSSL_INTEL_ASM) + # x86 only. Both the define and the -maes below are meaningless + # elsewhere, and -maes lands on CMAKE_C_FLAGS for the library AND every + # example and test target, so a cross compiler rejects the whole build. + # Fail with a clear message instead, as WOLFSSL_INTELASM_INTRINSICS and + # WOLFSSL_FALCON_ASM do below. + if(NOT WOLFSSL_X86_64_BUILD AND NOT WOLFSSL_X86_BUILD) + message(FATAL_ERROR + "WOLFSSL_AESNI / WOLFSSL_INTEL_ASM are x86 only") + endif() + set(WOLFSSL_AESNI "yes") + list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AESNI") + # aes.c uses AES-NI intrinsics directly in the key schedule, so the + # instructions have to be available to the compiler, not just to the + # assembler. A flag, not a define, so it goes on CMAKE_C_FLAGS rather + # than WOLFSSL_DEFINITIONS. This is an ISA the runtime dispatch already + # requires before calling in, so it does not widen what any dispatched + # routine may use. + if(NOT CMAKE_C_COMPILER_ID STREQUAL "MSVC") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes") + endif() +endif() + +# AES-XTS. The XTS assembly is wrapped in WOLFSSL_AES_XTS, so without this +# option the file is an empty translation unit. +add_option("WOLFSSL_AESXTS" + "Enable AES-XTS (default: disabled)" + "no" "yes;no") + +if(WOLFSSL_AESXTS) + list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AES_XTS") + # --enable-aesxts also turns the streaming API on by default for + # non-ARMASM builds (configure.ac defaults ENABLED_AESXTS_STREAM from + # ENABLED_AESXTS). Without this, cmake -DWOLFSSL_AESXTS=yes silently + # gives a smaller API than the autotools equivalent - no + # wc_AesXtsEncryptInit/Update/Final - which breaks the documented + # --enable-foo == -DWOLFSSL_FOO=yes mapping. + if(NOT WOLFSSL_ARM_ASM) + list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AESXTS_STREAM") + endif() +endif() + +# Build the Intel speedups from the generated C intrinsics rather than from +# the generated assembly. Both come out of one generator, so the two forms +# compute the same thing; the C form is visible to the optimiser, to LTO and +# to the sanitisers. Coverage is per algorithm - anything without an +# intrinsics translation keeps building from assembly. +# Falcon's floating-point backend. Falcon needs binary64 arithmetic that is +# bit-exact everywhere it runs, so the fpr layer is written in SSE2 rather +# than left to the compiler; this is autotools' --enable-falcon=asm. It is +# x86-64 only, and it selects a different fpr representation from the +# native-double backends, which is why those are mutually exclusive there. +add_option("WOLFSSL_FALCON_ASM" + "Build Falcon's fpr layer from the generated x86-64 assembly (default: disabled)" + "no" "yes;no") + +if (WOLFSSL_FALCON_ASM) + if (NOT WOLFSSL_FALCON) + message(FATAL_ERROR "WOLFSSL_FALCON_ASM requires WOLFSSL_FALCON") + endif() + if (NOT WOLFSSL_X86_64_BUILD) + message(FATAL_ERROR "WOLFSSL_FALCON_ASM is x86-64 only") + endif() + list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FALCON_FPR_ASM") + set_wolfssl_definitions("WOLFSSL_FALCON_FPR_ASM" RESULT) +endif() + +add_option("WOLFSSL_INTELASM_INTRINSICS" + "Build the x86-64 speedups from generated C intrinsics rather than assembly (default: disabled)" + "no" "yes;no") + +if(WOLFSSL_INTELASM_INTRINSICS) + if(NOT WOLFSSL_INTEL_ASM) + message(FATAL_ERROR + "WOLFSSL_INTELASM_INTRINSICS requires WOLFSSL_INTEL_ASM") + endif() + if(NOT WOLFSSL_X86_64_BUILD) + message(FATAL_ERROR "WOLFSSL_INTELASM_INTRINSICS is x86-64 only") + endif() + # Unlike the assembly it replaces, this code is compiled, and CMake adds + # no -O of its own unless a build type asks for one. At -O0 the generated + # SHA-256 runs several times slower than the assembly rather than at + # parity, which reads as the intrinsics being slow instead of unoptimised. + if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_C_FLAGS MATCHES "-O") + message(WARNING + "WOLFSSL_INTELASM_INTRINSICS with no CMAKE_BUILD_TYPE: the " + "generated C compiles unoptimised and will be far slower than " + "the assembly. Use -DCMAKE_BUILD_TYPE=Release.") + endif() +endif() + # SP math all add_option("WOLFSSL_SP_MATH_ALL" "Enable Single Precision math implementation for full algorithm suite (default: enabled)" @@ -4223,6 +4340,32 @@ endif() add_library(wolfssl::wolfssl ALIAS wolfssl) +# The generated x86-64 intrinsics are a literal transliteration of the assembly +# - one C statement per instruction - so their functions are enormous. +# Unoptimised, the compiler gives every intermediate its own stack slot and +# never reuses one: mlkem_encapsulate_avx512 gets a 718KB frame on gcc and +# 1.9MB on clang, enough to overflow a thread stack. At -O1 and above the slots +# are shared and the frame drops to about 2KB. CMake emits no -O at all when +# CMAKE_BUILD_TYPE is unset, which is the default, so force the level on just +# these sources - not on the rest of the library, whose flags stay the user's. +# Override with -DWOLFSSL_INTRIN_OPT=-O2 if you want a different level. +set(WOLFSSL_INTRIN_OPT "" CACHE STRING + "Optimisation flag for the generated *_intrin.c files (default -O1, /O1 on MSVC)") +if(NOT WOLFSSL_INTRIN_OPT) + if(MSVC) + set(WOLFSSL_INTRIN_OPT "/O1") + else() + set(WOLFSSL_INTRIN_OPT "-O1") + endif() +endif() +foreach(_intrin_src ${LIB_SOURCES}) + if(_intrin_src MATCHES "_intrin\\.c$") + set_source_files_properties("${_intrin_src}" PROPERTIES + COMPILE_OPTIONS "${WOLFSSL_INTRIN_OPT}") + endif() +endforeach() +unset(_intrin_src) + if (NOT "$ENV{ARIA_DIR}" STREQUAL "") message(STATUS "Found Environment variable ARIA_DIR=$ENV{ARIA_DIR}") if(WOLFSSL_ARIA) diff --git a/cmake/functions.cmake b/cmake/functions.cmake index 1856dcb22c3..eb4eee35218 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -124,7 +124,15 @@ function(generate_build_flags) set(BUILD_ARMASM ${WOLFSSL_ARM_ASM} PARENT_SCOPE) set(BUILD_XILINX ${WOLFSSL_XILINX} PARENT_SCOPE) set(BUILD_AESNI ${WOLFSSL_AESNI} PARENT_SCOPE) + set(BUILD_AESXTS ${WOLFSSL_AESXTS} PARENT_SCOPE) set(BUILD_INTELASM ${WOLFSSL_INTEL_ASM} PARENT_SCOPE) + set(BUILD_INTELASM_INTRINSICS ${WOLFSSL_INTELASM_INTRINSICS} PARENT_SCOPE) + set(BUILD_FALCON_ASM ${WOLFSSL_FALCON_ASM} PARENT_SCOPE) + # 32-bit x86 with the Intel speedups: selects the x86-specific AES-GCM + # assembly over the x86-64 one, as ENABLED_X86_ASM does in autotools. + if(WOLFSSL_X86_BUILD AND (WOLFSSL_AESNI OR WOLFSSL_INTEL_ASM)) + set(BUILD_X86_ASM "yes" PARENT_SCOPE) + endif() set(BUILD_AFALG ${WOLFSSL_AFALG} PARENT_SCOPE) set(BUILD_DEVCRYPTO ${WOLFSSL_DEVCRYPTO} PARENT_SCOPE) if(WOLFSSL_CAMELLIA OR WOLFSSL_USER_SETTINGS) @@ -468,10 +476,40 @@ function(generate_lib_src_list LIB_SOURCES) if(BUILD_AESNI) list(APPEND LIB_SOURCES wolfcrypt/src/aes_asm.S) + endif() + + # 32-bit x86 takes its own AES-GCM assembly and none of the + # x86-64 sources; everything below the else is 64-bit only. + if(BUILD_AESNI AND BUILD_X86_ASM) + list(APPEND LIB_SOURCES wolfcrypt/src/aes_gcm_x86_asm.S) + elseif(BUILD_AESNI) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/aes_x86_64_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/aes_x86_64_asm.S) + endif() + if(BUILD_AESXTS) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/aes_xts_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/aes_xts_asm.S) + endif() + endif() if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/aes_gcm_asm.S) - list(APPEND LIB_SOURCES wolfcrypt/src/sha3_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES wolfcrypt/src/aes_gcm_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/aes_gcm_asm.S) + endif() + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/sha3_x86_64_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/sha3_asm.S) + endif() elseif(BUILD_ARMASM) list(APPEND LIB_SOURCES wolfcrypt/src/port/arm/armv8-sha3-asm_c.c) list(APPEND LIB_SOURCES wolfcrypt/src/port/arm/armv8-sha3-asm.S) @@ -506,7 +544,10 @@ function(generate_lib_src_list LIB_SOURCES) endif() endif() - if(BUILD_INTELASM) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/sha256_x86_64_intrin.c) + elseif(BUILD_INTELASM) list(APPEND LIB_SOURCES wolfcrypt/src/sha256_asm.S) endif() @@ -533,7 +574,10 @@ function(generate_lib_src_list LIB_SOURCES) endif() endif() - if(BUILD_INTELASM) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/sha512_x86_64_intrin.c) + elseif(BUILD_INTELASM) list(APPEND LIB_SOURCES wolfcrypt/src/sha512_asm.S) endif() endif() @@ -562,7 +606,12 @@ function(generate_lib_src_list LIB_SOURCES) endif() if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/sha3_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/sha3_x86_64_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/sha3_asm.S) + endif() endif() endif() @@ -603,8 +652,16 @@ function(generate_lib_src_list LIB_SOURCES) wolfcrypt/src/wolfcrypt_first.c wolfcrypt/src/hmac.c wolfcrypt/src/random.c - wolfcrypt/src/sha256.c - wolfcrypt/src/sha256_asm.S + wolfcrypt/src/sha256.c) + + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/sha256_x86_64_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/sha256_asm.S) + endif() + + list(APPEND LIB_SOURCES wolfcrypt/src/fips.c wolfcrypt/src/fips_test.c wolfcrypt/src/wolfcrypt_last.c) @@ -666,7 +723,10 @@ function(generate_lib_src_list LIB_SOURCES) endif() else() - if(BUILD_INTELASM) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/sha256_x86_64_intrin.c) + elseif(BUILD_INTELASM) list(APPEND LIB_SOURCES wolfcrypt/src/sha256_asm.S) endif() endif() @@ -700,7 +760,13 @@ function(generate_lib_src_list LIB_SOURCES) if(BUILD_SP_X86_64) list(APPEND LIB_SOURCES wolfcrypt/src/sp_x86_64.c) if(WOLFSSL_X86_64_BUILD_ASM) - list(APPEND LIB_SOURCES wolfcrypt/src/sp_x86_64_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/sp_x86_64_intrin.c) + else() + list(APPEND LIB_SOURCES + wolfcrypt/src/sp_x86_64_asm.S) + endif() endif() endif() @@ -791,7 +857,10 @@ function(generate_lib_src_list LIB_SOURCES) endif() else() - if(BUILD_INTELASM) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/sha512_x86_64_intrin.c) + elseif(BUILD_INTELASM) list(APPEND LIB_SOURCES wolfcrypt/src/sha512_asm.S) endif() endif() @@ -821,7 +890,12 @@ function(generate_lib_src_list LIB_SOURCES) endif() if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/sha3_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/sha3_x86_64_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/sha3_asm.S) + endif() endif() endif() endif() @@ -887,7 +961,11 @@ function(generate_lib_src_list LIB_SOURCES) list(APPEND LIB_SOURCES wolfcrypt/src/poly1305.c) if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/poly1305_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES wolfcrypt/src/poly1305_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/poly1305_asm.S) + endif() endif() endif() @@ -914,9 +992,28 @@ function(generate_lib_src_list LIB_SOURCES) endif() if(NOT BUILD_FIPS_V2 AND BUILD_AESNI) - list(APPEND LIB_SOURCES - wolfcrypt/src/aes_asm.S - wolfcrypt/src/aes_gcm_asm.S) + list(APPEND LIB_SOURCES wolfcrypt/src/aes_asm.S) + # 32-bit x86 has its OWN AES-GCM assembly rather than a 32-bit + # build of the x86-64 one, and none of the other x86-64 sources + # apply there - the same split autotools makes on BUILD_X86_ASM. + if(BUILD_X86_ASM) + list(APPEND LIB_SOURCES wolfcrypt/src/aes_gcm_x86_asm.S) + else() + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES wolfcrypt/src/aes_gcm_intrin.c) + list(APPEND LIB_SOURCES wolfcrypt/src/aes_x86_64_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/aes_gcm_asm.S) + list(APPEND LIB_SOURCES wolfcrypt/src/aes_x86_64_asm.S) + endif() + if(BUILD_AESXTS) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES wolfcrypt/src/aes_xts_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/aes_xts_asm.S) + endif() + endif() + endif() endif() if(BUILD_CAMELLIA) @@ -1013,7 +1110,11 @@ function(generate_lib_src_list LIB_SOURCES) else() if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/chacha_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES wolfcrypt/src/chacha_intrin.c) + else() + list(APPEND LIB_SOURCES wolfcrypt/src/chacha_asm.S) + endif() endif() endif() @@ -1082,7 +1183,13 @@ function(generate_lib_src_list LIB_SOURCES) list(APPEND LIB_SOURCES wolfcrypt/src/fe_low_mem.c) else() if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/fe_x25519_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/fe_x25519_intrin.c) + else() + list(APPEND LIB_SOURCES + wolfcrypt/src/fe_x25519_asm.S) + endif() else() list(APPEND LIB_SOURCES wolfcrypt/src/fe_operations.c) endif() @@ -1101,7 +1208,13 @@ function(generate_lib_src_list LIB_SOURCES) if(NOT BUILD_FEMATH) if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/fe_x25519_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/fe_x25519_intrin.c) + else() + list(APPEND LIB_SOURCES + wolfcrypt/src/fe_x25519_asm.S) + endif() else() list(APPEND LIB_SOURCES wolfcrypt/src/fe_operations.c) endif() @@ -1136,13 +1249,29 @@ function(generate_lib_src_list LIB_SOURCES) if(BUILD_FALCON) list(APPEND LIB_SOURCES wolfcrypt/src/falcon.c) + + if(BUILD_FALCON_ASM) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/wc_falcon_fpr_intrin.c) + else() + list(APPEND LIB_SOURCES + wolfcrypt/src/wc_falcon_fpr_x86_64_asm.S) + endif() + endif() endif() if(BUILD_MLDSA) list(APPEND LIB_SOURCES wolfcrypt/src/wc_mldsa.c) if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/wc_mldsa_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/wc_mldsa_intrin.c) + else() + list(APPEND LIB_SOURCES + wolfcrypt/src/wc_mldsa_asm.S) + endif() endif() endif() @@ -1172,7 +1301,13 @@ function(generate_lib_src_list LIB_SOURCES) endif() if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/wc_mlkem_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/wc_mlkem_intrin.c) + else() + list(APPEND LIB_SOURCES + wolfcrypt/src/wc_mlkem_asm.S) + endif() endif() endif() @@ -1201,7 +1336,13 @@ function(generate_lib_src_list LIB_SOURCES) endif() if(BUILD_INTELASM) - list(APPEND LIB_SOURCES wolfcrypt/src/wc_frodokem_asm.S) + if(BUILD_INTELASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/wc_frodokem_intrin.c) + else() + list(APPEND LIB_SOURCES + wolfcrypt/src/wc_frodokem_asm.S) + endif() endif() endif() diff --git a/configure.ac b/configure.ac index 3b997f801ba..a09ff3ab822 100644 --- a/configure.ac +++ b/configure.ac @@ -4810,6 +4810,41 @@ fi AC_SUBST([ENABLED_AESNI]) AC_SUBST([ENABLED_AESNI_WITH_AVX]) +# INTEL ASM AS GENERATED C INTRINSICS +# Builds the generated intrinsics translation of the x86-64 speedups instead +# of the hand-generated assembly. Both come from one generator, so the two +# forms are interchangeable; the C form is visible to the optimiser, to LTO +# and to the sanitisers, which the .S is not. Coverage is per algorithm - +# anything without an intrinsics translation keeps building from assembly. +AC_ARG_ENABLE([intelasm-intrinsics], + [AS_HELP_STRING([--enable-intelasm-intrinsics],[Build the x86-64 speedups from generated C intrinsics rather than assembly (default: disabled)])], + [ ENABLED_INTELASM_INTRINSICS=$enableval ], + [ ENABLED_INTELASM_INTRINSICS=no ] + ) + +if test "$ENABLED_INTELASM_INTRINSICS" = "yes" +then + if test "$ENABLED_INTELASM" != "yes" + then + AC_MSG_ERROR([--enable-intelasm-intrinsics requires --enable-intelasm]) + fi + if test "$host_cpu" != "x86_64" && test "$host_cpu" != "amd64" + then + AC_MSG_ERROR([--enable-intelasm-intrinsics is x86-64 only]) + fi + # --enable-32bit leaves host_cpu as x86_64, so the test above passes for a + # -m32 build. WOLFSSL_X86_64_BUILD is deliberately withheld there (see the + # host_cpu block earlier in this file) and is not inert: it selects the + # Poly1305 AVX-512 state layout, the ChaCha and Curve25519 x64 paths, and + # the FP_MAX_BITS/SP_INT_BITS defaults. Reject the combination rather + # than build 32-bit code that claims to be 64-bit. + if test "$ENABLED_32BIT" = "yes" + then + AC_MSG_ERROR([--enable-intelasm-intrinsics is not supported with --enable-32bit]) + fi +fi +AC_SUBST([ENABLED_INTELASM_INTRINSICS]) + AC_ARG_ENABLE([aligndata], [AS_HELP_STRING([--enable-aligndata],[align data for ciphers (default: enabled)])], [ ENABLED_ALIGN_DATA=$enableval ], @@ -13266,6 +13301,7 @@ AM_CONDITIONAL([BUILD_PPC64_ASM_INLINE_REG],[test "x$ENABLED_PPC64_ASM_INLINE_RE AM_CONDITIONAL([BUILD_XILINX],[test "x$ENABLED_XILINX" = "xyes"]) AM_CONDITIONAL([BUILD_AESNI],[test "x$ENABLED_AESNI" = "xyes"]) AM_CONDITIONAL([BUILD_INTELASM],[test "x$ENABLED_INTELASM" = "xyes"]) +AM_CONDITIONAL([BUILD_INTELASM_INTRINSICS],[test "x$ENABLED_INTELASM_INTRINSICS" = "xyes"]) AM_CONDITIONAL([BUILD_X86_ASM],[test "x$ENABLED_X86_ASM" = "xyes"]) AM_CONDITIONAL([BUILD_AFALG],[test "x$ENABLED_AFALG" = "xyes"]) AM_CONDITIONAL([BUILD_KCAPI],[test "x$ENABLED_KCAPI" = "xyes"]) diff --git a/src/include.am b/src/include.am index 12133f62e33..36a09517fe6 100644 --- a/src/include.am +++ b/src/include.am @@ -53,6 +53,43 @@ src_libwolfssl@LIBSUFFIX@_la_LIBADD = $(LIBM) $(LIB_ADD) $(LIB_STATIC_ADD) src_libwolfssl@LIBSUFFIX@_la_CFLAGS = -DBUILDING_WOLFSSL $(AM_CFLAGS) -DLIBWOLFSSL_GLOBAL_EXTRA_CFLAGS="\" $(EXTRA_CFLAGS)\"" src_libwolfssl@LIBSUFFIX@_la_CPPFLAGS = -DBUILDING_WOLFSSL $(AM_CPPFLAGS) +# The generated x86-64 intrinsics (--enable-intelasm-intrinsics) are a literal +# transliteration of the assembly - one C statement per instruction - so their +# functions are enormous: mlkem_encapsulate_avx512 alone is 2511 intrinsic +# calls. Unoptimised, the compiler gives every intermediate its own stack slot +# and never reuses one, which puts that function's frame at 718KB on gcc and +# 1.9MB on clang; the 2MB thread stack the tests run on (--enable-stacksize) +# overflows. At -O1 and above the slots are shared and the frame drops to about +# 2KB. This is performance code, so an unoptimised build of it is not useful +# anyway, and a build that leaves CFLAGS empty (--enable-distro does) or sets +# -O0 must not silently produce something that overflows a thread stack. +# +# Force the level on just these objects. $(CFLAGS) is last on the compile line +# (see the generic .lo rule), so this wins over any -O in AM_CFLAGS without +# disturbing the flags of any other file. Override the level if you need to: +# make WOLFSSL_INTRIN_OPT=-O2 +# This is a GNU make target-specific variable. AM_INIT_AUTOMAKE already passes +# -Wno-portability, and include.am already uses GNU ':=' assignments below. +WOLFSSL_INTRIN_OPT = -O1 + +WOLFSSL_INTRIN_OBJS = \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-aes_gcm_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-aes_x86_64_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-aes_xts_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-chacha_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-fe_x25519_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-poly1305_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-sha256_x86_64_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-sha3_x86_64_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-sha512_x86_64_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-sp_x86_64_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-wc_falcon_fpr_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-wc_frodokem_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-wc_mldsa_intrin.lo \ + wolfcrypt/src/src_libwolfssl@LIBSUFFIX@_la-wc_mlkem_intrin.lo + +$(WOLFSSL_INTRIN_OBJS): CFLAGS += $(WOLFSSL_INTRIN_OPT) + if BUILD_FIPS_V5 LEGACY_ARMASM_AES_C := wolfcrypt/src/port/arm/armv8-aes.c LEGACY_ARMASM_SHA256_C := wolfcrypt/src/port/arm/armv8-sha256.c @@ -117,10 +154,18 @@ if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else if BUILD_AESGCM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif !BUILD_INTELASM_INTRINSICS endif if BUILD_AESXTS +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -135,7 +180,11 @@ endif if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif @@ -143,7 +192,11 @@ if BUILD_SHA512 src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512.c if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -152,7 +205,11 @@ if BUILD_SHA3 src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3.c if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -181,8 +238,13 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += \ wolfcrypt/src/wolfcrypt_first.c \ wolfcrypt/src/hmac.c \ wolfcrypt/src/random.c \ - wolfcrypt/src/sha256.c \ - wolfcrypt/src/sha256_asm.S \ + wolfcrypt/src/sha256.c +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_x86_64_intrin.c +else +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_asm.S +endif !BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += \ wolfcrypt/src/fips.c \ wolfcrypt/src/fips_test.c \ wolfcrypt/src/wolfcrypt_last.c @@ -279,10 +341,18 @@ if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else if BUILD_AESGCM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif !BUILD_INTELASM_INTRINSICS endif if BUILD_AESXTS +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -330,7 +400,11 @@ else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256.c if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_X86_ASM endif !BUILD_ARMASM @@ -414,7 +488,11 @@ endif !BUILD_FIPS_V5 if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_X86_ASM endif !BUILD_ARMASM @@ -492,7 +570,11 @@ endif !BUILD_PPC32_ASM_INLINE endif BUILD_PPC32_ASM if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -615,10 +697,18 @@ if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else if BUILD_AESGCM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif !BUILD_INTELASM_INTRINSICS endif if BUILD_AESXTS +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -664,7 +754,11 @@ endif !BUILD_ARMASM_INLINE else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256.c if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_ARMASM endif !BUILD_ARMASM_NEON @@ -745,7 +839,11 @@ endif BUILD_FIPS_V6 endif !BUILD_FIPS_V5 if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_ARMASM endif !BUILD_ARMASM_NEON @@ -821,7 +919,11 @@ endif !BUILD_PPC32_ASM_INLINE_REG endif !BUILD_PPC32_ASM_INLINE endif BUILD_PPC32_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif @@ -905,7 +1007,11 @@ endif if BUILD_SP_X86_64 src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sp_x86_64.c +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sp_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sp_x86_64_asm.S +endif !BUILD_INTELASM_INTRINSICS endif if !BUILD_FIPS_V2 if BUILD_SP_ARM32 @@ -1002,15 +1108,27 @@ endif BUILD_AES if BUILD_AESNI src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_x86_64_asm.S +endif !BUILD_INTELASM_INTRINSICS if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else if BUILD_AESGCM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif !BUILD_INTELASM_INTRINSICS endif if BUILD_AESXTS +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -1051,7 +1169,11 @@ endif !BUILD_ARMASM_INLINE else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256.c if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_ARMASM endif !BUILD_ARMASM_NEON @@ -1132,7 +1254,11 @@ endif BUILD_FIPS_V6 endif !BUILD_FIPS_V5 if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_ARMASM endif !BUILD_ARMASM_NEON @@ -1208,7 +1334,11 @@ endif !BUILD_PPC32_ASM_INLINE_REG endif !BUILD_PPC32_ASM_INLINE endif BUILD_PPC32_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif @@ -1288,7 +1418,11 @@ endif if BUILD_SP_X86_64 src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sp_x86_64.c +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sp_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sp_x86_64_asm.S +endif !BUILD_INTELASM_INTRINSICS endif if !BUILD_FIPS_V2 if BUILD_SP_ARM32 @@ -1332,7 +1466,11 @@ endif !BUILD_ARM_THUMB endif BUILD_ARMASM if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mlkem_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mlkem_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif if BUILD_ARMASM_NEON @@ -1348,7 +1486,11 @@ if BUILD_MLDSA src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mldsa.c if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mldsa_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mldsa_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_X86_ASM endif @@ -1461,7 +1603,11 @@ endif !BUILD_ARMASM_INLINE else if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha256_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_X86_ASM endif !BUILD_ARMASM @@ -1536,7 +1682,11 @@ endif if BUILD_SP_X86_64 src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sp_x86_64.c +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sp_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sp_x86_64_asm.S +endif !BUILD_INTELASM_INTRINSICS endif if !BUILD_FIPS_V2 if BUILD_SP_ARM32 @@ -1719,7 +1869,11 @@ endif !BUILD_FIPS_V5 if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha512_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_X86_ASM endif !BUILD_ARMASM @@ -1799,7 +1953,11 @@ endif !BUILD_PPC32_ASM_INLINE endif BUILD_PPC32_ASM if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/sha3_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -1875,7 +2033,11 @@ if BUILD_WC_FRODOKEM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_frodokem.c src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_frodokem_mat.c if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_frodokem_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_frodokem_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM # AArch64: NEON, plus the opt-in SVE and SME ops (guarded by WOLFSSL_FRODOKEM_SVE # / WOLFSSL_FRODOKEM_SME), all in the one armv8-frodokem-asm file. @@ -1974,7 +2136,11 @@ endif src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/poly1305.c if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/poly1305_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/poly1305_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -2008,15 +2174,27 @@ endif if !BUILD_FIPS_V2_PLUS if BUILD_AESNI src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_x86_64_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_x86_64_asm.S +endif !BUILD_INTELASM_INTRINSICS if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S else if BUILD_AESGCM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S +endif !BUILD_INTELASM_INTRINSICS endif if BUILD_AESXTS +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif endif @@ -2069,7 +2247,11 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/riscv64/riscv-64-chac endif BUILD_RISCV_ASM if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/chacha_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/chacha_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_X86_ASM endif !BUILD_ARMASM @@ -2133,7 +2315,11 @@ endif !BUILD_ARM_THUMB endif BUILD_ARMASM if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mlkem_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mlkem_asm.S +endif !BUILD_INTELASM_INTRINSICS endif endif if BUILD_ARMASM_NEON @@ -2149,7 +2335,11 @@ if BUILD_MLDSA src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mldsa.c if !BUILD_X86_ASM if BUILD_INTELASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mldsa_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mldsa_asm.S +endif !BUILD_INTELASM_INTRINSICS endif BUILD_INTELASM endif !BUILD_X86_ASM endif @@ -2186,7 +2376,11 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/fe_low_mem.c src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/ge_operations.c if BUILD_CURVE25519_INTELASM if !BUILD_X86_ASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/fe_x25519_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/fe_x25519_asm.S +endif !BUILD_INTELASM_INTRINSICS else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/fe_operations.c endif BUILD_X86_ASM @@ -2249,7 +2443,11 @@ if !BUILD_FEMATH src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/ge_operations.c if BUILD_CURVE25519_INTELASM if !BUILD_X86_ASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/fe_x25519_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/fe_x25519_asm.S +endif !BUILD_INTELASM_INTRINSICS else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/fe_operations.c endif BUILD_X86_ASM @@ -2313,7 +2511,11 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/falcon.c endif if BUILD_FALCON_ASM +if BUILD_INTELASM_INTRINSICS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_falcon_fpr_intrin.c +else src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_falcon_fpr_x86_64_asm.S +endif !BUILD_INTELASM_INTRINSICS endif diff --git a/wolfcrypt/src/aes_gcm_intrin.c b/wolfcrypt/src/aes_gcm_intrin.c new file mode 100644 index 00000000000..c3e221167fd --- /dev/null +++ b/wolfcrypt/src/aes_gcm_intrin.c @@ -0,0 +1,45216 @@ +/* aes_gcm_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_AES_GCM_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_X86_64_BUILD +extern WOLFSSL_LOCAL void GCM_generate_m0_aesni(const unsigned char* h, + unsigned char* m); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_aesni(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_aesni(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res); +#ifdef WOLFSSL_AESGCM_STREAM +extern WOLFSSL_LOCAL void AES_GCM_init_aesni(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_aad_update_aesni(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_block_aesni(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, + unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_ghash_block_aesni(const unsigned char* addt, + unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_update_aesni(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_final_aesni(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_update_aesni(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_final_aesni(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res); +#endif +#ifdef WOLFSSL_AESGCM_SIV +extern WOLFSSL_LOCAL void AES_GCMSIV_polyval_aesni(unsigned char* s, + const unsigned char* h, const unsigned char* data, unsigned int blocks); +extern WOLFSSL_LOCAL void AES_GCMSIV_ctr_aesni(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr); +#endif +#ifdef HAVE_INTEL_AVX1 +extern WOLFSSL_LOCAL void GCM_generate_m0_avx1(const unsigned char* h, + unsigned char* m); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_avx1(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_avx1(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res); +#ifdef WOLFSSL_AESGCM_STREAM +extern WOLFSSL_LOCAL void AES_GCM_init_avx1(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_aad_update_avx1(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_block_avx1(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, + unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_ghash_block_avx1(const unsigned char* addt, + unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_update_avx1(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_final_avx1(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_update_avx1(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_final_avx1(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res); +#endif +#ifdef WOLFSSL_AESGCM_SIV +extern WOLFSSL_LOCAL void AES_GCMSIV_polyval_avx1(unsigned char* s, + const unsigned char* h, const unsigned char* data, unsigned int blocks); +extern WOLFSSL_LOCAL void AES_GCMSIV_ctr_avx1(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr); +#endif +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void GCM_generate_m0_avx2(const unsigned char* h, + unsigned char* m); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_avx2(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_avx2(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res); +#ifdef WOLFSSL_AESGCM_STREAM +extern WOLFSSL_LOCAL void AES_GCM_init_avx2(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_aad_update_avx2(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_block_avx2(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, + unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_ghash_block_avx2(const unsigned char* addt, + unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_update_avx2(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_final_avx2(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_update_avx2(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_final_avx2(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res); +#endif +#endif +#ifdef HAVE_INTEL_VAES +extern WOLFSSL_LOCAL void GCM_generate_m0_vaes(const unsigned char* h, + unsigned char* m); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_vaes(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_vaes(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res); +#ifdef WOLFSSL_AESGCM_STREAM +extern WOLFSSL_LOCAL void AES_GCM_init_vaes(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_aad_update_vaes(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_block_vaes(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, + unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_ghash_block_vaes(const unsigned char* addt, + unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_update_vaes(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_final_vaes(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_update_vaes(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_final_vaes(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res); +#endif +#ifdef WOLFSSL_AESGCM_SIV +extern WOLFSSL_LOCAL void AES_GCMSIV_polyval_vaes(unsigned char* s, + const unsigned char* h, const unsigned char* data, unsigned int blocks); +extern WOLFSSL_LOCAL void AES_GCMSIV_ctr_vaes(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr); +#endif +#endif +#ifdef HAVE_INTEL_AVX512 +extern WOLFSSL_LOCAL void GCM_generate_m0_avx512(const unsigned char* h, + unsigned char* m); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_avx512(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_avx512(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res); +#ifdef WOLFSSL_AESGCM_STREAM +extern WOLFSSL_LOCAL void AES_GCM_init_avx512(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_aad_update_avx512(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_block_avx512(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, + unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_ghash_block_avx512(const unsigned char* addt, + unsigned char* tag, unsigned char* h); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_update_avx512( + const unsigned char* key, int nr, unsigned char* out, + const unsigned char* in, unsigned int nbytes, unsigned char* tag, + unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_encrypt_final_avx512(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_update_avx512( + const unsigned char* key, int nr, unsigned char* out, + const unsigned char* in, unsigned int nbytes, unsigned char* tag, + unsigned char* h, unsigned char* counter); +extern WOLFSSL_LOCAL void AES_GCM_decrypt_final_avx512(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res); +#endif +#ifdef WOLFSSL_AESGCM_SIV +extern WOLFSSL_LOCAL void AES_GCMSIV_polyval_avx512(unsigned char* s, + const unsigned char* h, const unsigned char* data, unsigned int blocks); +extern WOLFSSL_LOCAL void AES_GCMSIV_ctr_avx512(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr); + +#endif +#endif +#endif +#ifdef WOLFSSL_X86_64_BUILD +XALIGNED(16) static const word64 L_GCM_generate_m0_aesni_rev8[] + WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_GCM_generate_m0_aesni_mod2_128[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0xe100000000000000ULL, +}; + +WC_X64I_TARGET("sse2,ssse3") +WOLFSSL_LOCAL void GCM_generate_m0_aesni(const unsigned char* h, + unsigned char* m) +{ + word64 rdi, rsi; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10; + + rdi = (word64)(size_t)h; + rsi = (word64)(size_t)m; + + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_GCM_generate_m0_aesni_rev8, + 0)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR( + L_GCM_generate_m0_aesni_mod2_128, 0)); + x8 = _mm_setzero_si128(); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x8); + x8 = x0; + x0 = _mm_shuffle_epi8(x0, x9); + x5 = x0; + x4 = x0; + x5 = _mm_slli_epi64(x5, 63); + x4 = _mm_srli_epi64(x4, 1); + x1 = x5; + x1 = _mm_bslli_si128(x1, 8); + x5 = _mm_bsrli_si128(x5, 8); + x1 = _mm_shuffle_epi32(x1, 0xff); + x4 = _mm_or_si128(x4, x5); + x1 = _mm_srai_epi32(x1, 31); + x1 = _mm_and_si128(x1, x10); + x1 = _mm_xor_si128(x1, x4); + x5 = x1; + x4 = x1; + x5 = _mm_slli_epi64(x5, 63); + x4 = _mm_srli_epi64(x4, 1); + x2 = x5; + x2 = _mm_bslli_si128(x2, 8); + x5 = _mm_bsrli_si128(x5, 8); + x2 = _mm_shuffle_epi32(x2, 0xff); + x4 = _mm_or_si128(x4, x5); + x2 = _mm_srai_epi32(x2, 31); + x2 = _mm_and_si128(x2, x10); + x2 = _mm_xor_si128(x2, x4); + x5 = x2; + x4 = x2; + x5 = _mm_slli_epi64(x5, 63); + x4 = _mm_srli_epi64(x4, 1); + x3 = x5; + x3 = _mm_bslli_si128(x3, 8); + x5 = _mm_bsrli_si128(x5, 8); + x3 = _mm_shuffle_epi32(x3, 0xff); + x4 = _mm_or_si128(x4, x5); + x3 = _mm_srai_epi32(x3, 31); + x3 = _mm_and_si128(x3, x10); + x3 = _mm_xor_si128(x3, x4); + x3 = _mm_shuffle_epi8(x3, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x8 = x3; + x1 = _mm_shuffle_epi8(x1, x9); + x0 = _mm_shuffle_epi8(x0, x9); + x8 = _mm_xor_si128(x8, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x3); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x1); + x4 = x3; + x5 = x2; + x6 = x8; + x4 = _mm_xor_si128(x4, x1); + x5 = _mm_xor_si128(x5, x1); + x6 = _mm_xor_si128(x6, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 128), x0); + x1 = _mm_xor_si128(x1, x0); + x4 = x3; + x6 = x2; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 144), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 160), x6); + x6 = _mm_xor_si128(x6, x3); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 176), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 192), x1); + x4 = x3; + x5 = x2; + x6 = x8; + x4 = _mm_xor_si128(x4, x1); + x5 = _mm_xor_si128(x5, x1); + x6 = _mm_xor_si128(x6, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 208), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 224), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 240), x6); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = x0; + x5 = x1; + x6 = x2; + x7 = x3; + x4 = _mm_slli_epi64(x4, 60); + x5 = _mm_slli_epi64(x5, 60); + x6 = _mm_slli_epi64(x6, 60); + x7 = _mm_slli_epi64(x7, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 256), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 272), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 288), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 304), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = x0; + x5 = x1; + x6 = x2; + x7 = x3; + x4 = _mm_slli_epi64(x4, 60); + x5 = _mm_slli_epi64(x5, 60); + x6 = _mm_slli_epi64(x6, 60); + x7 = _mm_slli_epi64(x7, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 320), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 336), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 352), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 368), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = x0; + x5 = x1; + x6 = x2; + x7 = x3; + x4 = _mm_slli_epi64(x4, 60); + x5 = _mm_slli_epi64(x5, 60); + x6 = _mm_slli_epi64(x6, 60); + x7 = _mm_slli_epi64(x7, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 384), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 400), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 416), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 432), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 240)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = x0; + x5 = x1; + x6 = x2; + x7 = x3; + x4 = _mm_slli_epi64(x4, 60); + x5 = _mm_slli_epi64(x5, 60); + x6 = _mm_slli_epi64(x6, 60); + x7 = _mm_slli_epi64(x7, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 448), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 464), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 480), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 496), x3); +} + +XALIGNED(16) static const word64 L_aes_gcm_one[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000001ULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_two[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_three[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000003ULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_four[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_five[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000005ULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_six[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000006ULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_seven[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000007ULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_eight[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000008ULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_bswap_epi64[] WC_X64I_UNUSED = { + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_bswap_mask[] WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_aes_gcm_mod2_128[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0xc200000000000000ULL, +}; + +WC_X64I_TARGET("sse2,ssse3,sse4.1,aes,pclmul") +WOLFSSL_LOCAL void AES_GCM_encrypt_aesni(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr) +{ + word64 rdi, rsi, r12, rax, r8, r9, r11, rbx, r14, r15, r10, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6, x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[24]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + word32 zf35; + word32 zf36; + word32 zf37; + word32 zf38; + word32 zf39; + word32 zf40; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r8 = (word64)(size_t)tag; + r9 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r14 = (word64)(word64)tbytes; + r15 = (word64)(size_t)key; + r10 = (word64)(word64)nr; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 160); + x4 = _mm_setzero_si128(); + x6 = _mm_setzero_si128(); + zf1 = (word32)rbx; + zf2 = 0xc; + rdx = (word32)((word32)rbx); + if ((zf1) != (zf2)) { + goto L_AES_GCM_encrypt_aesni_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_insert_epi64(x4, (long long)WC_L64(rax, 0), 0); + x4 = _mm_insert_epi32(x4, (int)WC_L32(rax, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x1 = x4; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x1 = _mm_xor_si128(x1, x5); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + zf3 = (word32)r10; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + zf5 = (word32)r10; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_aesni_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x7); + x1 = _mm_aesenclast_si128(x1, x7); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x1); + goto L_AES_GCM_encrypt_aesni_iv_done; +L_AES_GCM_encrypt_aesni_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf7 = (word32)r10; + zf8 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x9); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf9 = (word32)r10; + zf10 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x9); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_aesni_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x9); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf11 = (word32)rdx; + zf12 = 0; + rcx = (word64)(0); + if ((zf11) == (zf12)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_encrypt_aesni_calc_iv_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rax, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x7; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_done; + } +L_AES_GCM_encrypt_aesni_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_encrypt_aesni_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x7; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); +L_AES_GCM_encrypt_aesni_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_insert_epi64(x0, (long long)rdx, 0); + x4 = _mm_xor_si128(x4, x0); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x7; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x8 = _mm_xor_si128(x8, x4); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf13 = (word32)r10; + zf14 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf15 = (word32)r10; + zf16 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_aesni_calc_iv_2_aesenc_avx_last: + x8 = _mm_aesenclast_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x8); +L_AES_GCM_encrypt_aesni_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_encrypt_aesni_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_encrypt_aesni_calc_aad_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_encrypt_aesni_calc_aad_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + x1 = _mm_shuffle_epi32(x6, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x6, 0x11); + x0 = _mm_clmulepi64_si128(x0, x6, 0); + x1 = _mm_xor_si128(x1, x6); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = x7; + x1 = x6; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x6 = _mm_xor_si128(x6, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_aesni_calc_aad_16_loop; + } + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_aesni_calc_aad_done; + } +L_AES_GCM_encrypt_aesni_calc_aad_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_encrypt_aesni_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_aesni_calc_aad_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + x1 = _mm_shuffle_epi32(x6, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x6, 0x11); + x0 = _mm_clmulepi64_si128(x0, x6, 0); + x1 = _mm_xor_si128(x1, x6); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = x7; + x1 = x6; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x6 = _mm_xor_si128(x6, x2); +L_AES_GCM_encrypt_aesni_calc_aad_done: + /* Calculate counter and H */ + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x9 = x5; + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x8 = x5; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x4); + x9 = _mm_srli_epi64(x9, 63); + x8 = _mm_slli_epi64(x8, 1); + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_or_si128(x8, x9); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + rbx = (word64)(0); + zf17 = (word32)r9; + zf18 = 0x80; + r13 = (word32)((word32)r9); + if ((zf17) < (zf18)) { + goto L_AES_GCM_encrypt_aesni_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + x2 = x6; + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x5, 0x4e); + x11 = x5; + x8 = x5; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x5); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x0 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x0 = _mm_xor_si128(x0, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x0 = _mm_xor_si128(x0, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = x0; + x8 = x0; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x1 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x1 = _mm_xor_si128(x1, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x1 = _mm_xor_si128(x1, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + /* H ^ 4 */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = x0; + x8 = x0; + x11 = _mm_clmulepi64_si128(x11, x0, 0x11); + x8 = _mm_clmulepi64_si128(x8, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x3 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x3 = _mm_xor_si128(x3, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x3 = _mm_xor_si128(x3, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x3); + /* H ^ 5 */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = x1; + x8 = x1; + x11 = _mm_clmulepi64_si128(x11, x0, 0x11); + x8 = _mm_clmulepi64_si128(x8, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = x1; + x8 = x1; + x11 = _mm_clmulepi64_si128(x11, x1, 0x11); + x8 = _mm_clmulepi64_si128(x8, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = x3; + x8 = x3; + x11 = _mm_clmulepi64_si128(x11, x1, 0x11); + x8 = _mm_clmulepi64_si128(x8, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x9 = _mm_shuffle_epi32(x3, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = x3; + x8 = x3; + x11 = _mm_clmulepi64_si128(x11, x3, 0x11); + x8 = _mm_clmulepi64_si128(x8, x3, 0); + x9 = _mm_xor_si128(x9, x3); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + /* First 128 bytes of input */ + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_bswap_epi64, 0)); + x0 = x8; + x8 = _mm_shuffle_epi8(x8, x1); + x9 = x0; + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = x0; + x10 = _mm_add_epi32(x10, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = x0; + x11 = _mm_add_epi32(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = x0; + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = x0; + x13 = _mm_add_epi32(x13, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = x0; + x14 = _mm_add_epi32(x14, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = x0; + x15 = _mm_add_epi32(x15, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf19 = (word32)r10; + zf20 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_encrypt_aesni_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf21 = (word32)r10; + zf22 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_encrypt_aesni_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_aesni_enc_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x15); + zf23 = (word32)r13; + zf24 = 0x80; + rbx = (word32)(0x80); + if ((zf23) <= (zf24)) { + goto L_AES_GCM_encrypt_aesni_end_128; + } + /* More 128 bytes of input */ +L_AES_GCM_encrypt_aesni_ghash_128: + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_bswap_epi64, 0)); + x0 = x8; + x8 = _mm_shuffle_epi8(x8, x1); + x9 = x0; + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = x0; + x10 = _mm_add_epi32(x10, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = x0; + x11 = _mm_add_epi32(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = x0; + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = x0; + x13 = _mm_add_epi32(x13, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = x0; + x14 = _mm_add_epi32(x14, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = x0; + x15 = _mm_add_epi32(x15, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -128)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x2); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x1 = _mm_xor_si128(x1, x7); + x5 = _mm_xor_si128(x5, x0); + x3 = x0; + x3 = _mm_clmulepi64_si128(x3, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x2 = x0; + x2 = _mm_clmulepi64_si128(x2, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x1 = _mm_clmulepi64_si128(x1, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x3); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -112)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -96)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -80)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -64)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -48)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -32)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -16)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x5 = x1; + x1 = _mm_bsrli_si128(x1, 8); + x5 = _mm_bslli_si128(x5, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x1); + x7 = x2; + x4 = x2; + x5 = x2; + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x7 = _mm_slli_epi32(x7, 31); + x4 = _mm_slli_epi32(x4, 30); + x5 = _mm_slli_epi32(x5, 25); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_xor_si128(x7, x5); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x4 = x7; + x7 = _mm_bslli_si128(x7, 12); + x4 = _mm_bsrli_si128(x4, 4); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x2 = _mm_xor_si128(x2, x7); + x5 = x2; + x1 = x2; + x0 = x2; + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_srli_epi32(x5, 1); + x1 = _mm_srli_epi32(x1, 2); + x0 = _mm_srli_epi32(x0, 7); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_xor_si128(x5, x1); + x5 = _mm_xor_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_xor_si128(x5, x4); + x2 = _mm_xor_si128(x2, x5); + x2 = _mm_xor_si128(x2, x3); + zf25 = (word32)r10; + zf26 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf27 = (word32)r10; + zf28 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_aesni_aesenc_128_ghash_avx_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + rbx = (word32)((word32)rbx + 0x80); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_aesni_ghash_128; + } +L_AES_GCM_encrypt_aesni_end_128: + x4 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_bswap_mask, 0)); + x8 = _mm_shuffle_epi8(x8, x4); + x9 = _mm_shuffle_epi8(x9, x4); + x10 = _mm_shuffle_epi8(x10, x4); + x11 = _mm_shuffle_epi8(x11, x4); + x8 = _mm_xor_si128(x8, x2); + x12 = _mm_shuffle_epi8(x12, x4); + x13 = _mm_shuffle_epi8(x13, x4); + x14 = _mm_shuffle_epi8(x14, x4); + x15 = _mm_shuffle_epi8(x15, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x1 = _mm_shuffle_epi32(x8, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x8, 0x11); + x0 = _mm_clmulepi64_si128(x0, x8, 0); + x1 = _mm_xor_si128(x1, x8); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x1 = _mm_shuffle_epi32(x9, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x9, 0x11); + x0 = _mm_clmulepi64_si128(x0, x9, 0); + x1 = _mm_xor_si128(x1, x9); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x1 = _mm_shuffle_epi32(x10, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x10, 0x11); + x0 = _mm_clmulepi64_si128(x0, x10, 0); + x1 = _mm_xor_si128(x1, x10); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x1 = _mm_shuffle_epi32(x11, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x11, 0x11); + x0 = _mm_clmulepi64_si128(x0, x11, 0); + x1 = _mm_xor_si128(x1, x11); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x1 = _mm_shuffle_epi32(x12, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x12, 0x11); + x0 = _mm_clmulepi64_si128(x0, x12, 0); + x1 = _mm_xor_si128(x1, x12); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x1 = _mm_shuffle_epi32(x13, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x13, 0x11); + x0 = _mm_clmulepi64_si128(x0, x13, 0); + x1 = _mm_xor_si128(x1, x13); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x1 = _mm_shuffle_epi32(x14, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x14, 0x11); + x0 = _mm_clmulepi64_si128(x0, x14, 0); + x1 = _mm_xor_si128(x1, x14); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x1 = _mm_shuffle_epi32(x15, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x15, 0x11); + x0 = _mm_clmulepi64_si128(x0, x15, 0); + x1 = _mm_xor_si128(x1, x15); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = x4; + x1 = x4; + x2 = x4; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x4 = _mm_xor_si128(x4, x0); + x2 = x4; + x3 = x4; + x0 = x4; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x4); + x6 = _mm_xor_si128(x6, x2); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_encrypt_aesni_done_128: + rdx = (word32)((word32)r9); + if (((word32)rbx) >= ((word32)rdx)) { + goto L_AES_GCM_encrypt_aesni_done_enc; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_aesni_last_block_done; + } + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x9 = x8; + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf29 = (word32)r10; + zf30 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf31 = (word32)r10; + zf32 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_aesni_aesenc_block_aesenc_avx_last: + x8 = _mm_aesenclast_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_aesni_last_block_ghash; + } +L_AES_GCM_encrypt_aesni_last_block_start: + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x9 = x8; + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x9); + x10 = x6; + x10 = _mm_clmulepi64_si128(x10, x5, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x11 = x6; + x11 = _mm_clmulepi64_si128(x11, x5, 1); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x12 = x6; + x12 = _mm_clmulepi64_si128(x12, x5, 0); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x1 = x6; + x1 = _mm_clmulepi64_si128(x1, x5, 0x11); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x10 = _mm_xor_si128(x10, x11); + x2 = x10; + x10 = _mm_bsrli_si128(x10, 8); + x2 = _mm_bslli_si128(x2, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x3 = x1; + x2 = _mm_xor_si128(x2, x12); + x3 = _mm_xor_si128(x3, x10); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_mod2_128, 0)); + x11 = x2; + x11 = _mm_clmulepi64_si128(x11, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x10 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x11 = x10; + x11 = _mm_clmulepi64_si128(x11, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x6 = _mm_shuffle_epi32(x10, 0x4e); + x6 = _mm_xor_si128(x6, x11); + x6 = _mm_xor_si128(x6, x3); + zf33 = (word32)r10; + zf34 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_GCM_encrypt_aesni_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf35 = (word32)r10; + zf36 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf35) < (sword32)(zf36)) { + goto L_AES_GCM_encrypt_aesni_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_aesni_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_aesni_last_block_start; + } +L_AES_GCM_encrypt_aesni_last_block_ghash: + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = x6; + x8 = x6; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x6 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x6, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); +L_AES_GCM_encrypt_aesni_last_block_done: + rcx = (word32)((word32)r9); + rdx = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x4 = _mm_xor_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf37 = (word32)r10; + zf38 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf37) < (sword32)(zf38)) { + goto L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x9); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf39 = (word32)r10; + zf40 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf39) < (sword32)(zf40)) { + goto L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x9); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_aesenc_avx_last: + x4 = _mm_aesenclast_si128(x4, x9); + rsp = (word64)(rsp - 16); + rcx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(rsi, rbx) = (byte)((byte)r13); + WC_S8(rsp, rcx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_loop; + } + r13 = (word64)(0); + if (((word32)rcx) == (0x10)) { + goto L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc; + } +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop: + WC_S8(rsp, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) < (0x10)) { + goto L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_byte_loop; + } +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_finish_enc: + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x4); + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = x6; + x8 = x6; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x6 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x6, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); +L_AES_GCM_encrypt_aesni_aesenc_last15_enc_avx_done: +L_AES_GCM_encrypt_aesni_done_enc: + rdx = (word32)((word32)r9); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_insert_epi64(x0, (long long)rdx, 0); + x0 = _mm_insert_epi64(x0, (long long)rcx, 1); + x6 = _mm_xor_si128(x6, x0); + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = x6; + x8 = x6; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x6 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x6, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + x0 = _mm_xor_si128(x0, x6); + if (((word32)r14) == (0x10)) { + goto L_AES_GCM_encrypt_aesni_store_tag_16; + } + rcx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_aesni_store_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + WC_S8(r8, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)r14)) { + goto L_AES_GCM_encrypt_aesni_store_tag_loop; + } + goto L_AES_GCM_encrypt_aesni_store_tag_done; +L_AES_GCM_encrypt_aesni_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x0); +L_AES_GCM_encrypt_aesni_store_tag_done: + ; +} + +WC_X64I_TARGET("sse2,ssse3,sse4.1,aes,pclmul") +WOLFSSL_LOCAL void AES_GCM_decrypt_aesni(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res) +{ + word64 rdi, rsi, r12, rax, r8, r9, r11, rbx, r14, r15, r10, rbp, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6, x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[28]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r8 = (word64)(size_t)tag; + r9 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r14 = (word64)(word64)tbytes; + r15 = (word64)(size_t)key; + r10 = (word64)(word64)nr; + rbp = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 224; + rsp = (word64)(rsp - 168); + x4 = _mm_setzero_si128(); + x6 = _mm_setzero_si128(); + zf1 = (word32)rbx; + zf2 = 0xc; + rdx = (word32)((word32)rbx); + if ((zf1) != (zf2)) { + goto L_AES_GCM_decrypt_aesni_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_insert_epi64(x4, (long long)WC_L64(rax, 0), 0); + x4 = _mm_insert_epi32(x4, (int)WC_L32(rax, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x1 = x4; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x1 = _mm_xor_si128(x1, x5); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + zf3 = (word32)r10; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + zf5 = (word32)r10; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_aesni_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x7); + x1 = _mm_aesenclast_si128(x1, x7); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x1); + goto L_AES_GCM_decrypt_aesni_iv_done; +L_AES_GCM_decrypt_aesni_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf7 = (word32)r10; + zf8 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x9); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf9 = (word32)r10; + zf10 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x9); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_aesni_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x9); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf11 = (word32)rdx; + zf12 = 0; + rcx = (word64)(0); + if ((zf11) == (zf12)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_decrypt_aesni_calc_iv_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rax, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x7; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_done; + } +L_AES_GCM_decrypt_aesni_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_decrypt_aesni_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x7; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); +L_AES_GCM_decrypt_aesni_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_insert_epi64(x0, (long long)rdx, 0); + x4 = _mm_xor_si128(x4, x0); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x7; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x8 = _mm_xor_si128(x8, x4); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf13 = (word32)r10; + zf14 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf15 = (word32)r10; + zf16 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_aesni_calc_iv_2_aesenc_avx_last: + x8 = _mm_aesenclast_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x8); +L_AES_GCM_decrypt_aesni_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_decrypt_aesni_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_decrypt_aesni_calc_aad_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_decrypt_aesni_calc_aad_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + x1 = _mm_shuffle_epi32(x6, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x6, 0x11); + x0 = _mm_clmulepi64_si128(x0, x6, 0); + x1 = _mm_xor_si128(x1, x6); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = x7; + x1 = x6; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x6 = _mm_xor_si128(x6, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_aesni_calc_aad_16_loop; + } + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_aesni_calc_aad_done; + } +L_AES_GCM_decrypt_aesni_calc_aad_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_decrypt_aesni_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_aesni_calc_aad_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + x1 = _mm_shuffle_epi32(x6, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x6, 0x11); + x0 = _mm_clmulepi64_si128(x0, x6, 0); + x1 = _mm_xor_si128(x1, x6); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x7 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = x7; + x1 = x6; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + x0 = x7; + x1 = x7; + x2 = x7; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = x7; + x3 = x7; + x0 = x7; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x6 = _mm_xor_si128(x6, x2); +L_AES_GCM_decrypt_aesni_calc_aad_done: + /* Calculate counter and H */ + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x9 = x5; + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x8 = x5; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x4); + x9 = _mm_srli_epi64(x9, 63); + x8 = _mm_slli_epi64(x8, 1); + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_or_si128(x8, x9); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + rbx = (word32)(0); + zf17 = (word32)r9; + zf18 = 0x80; + r13 = (word32)((word32)r9); + if ((zf17) < (zf18)) { + goto L_AES_GCM_decrypt_aesni_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + x2 = x6; + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x5, 0x4e); + x11 = x5; + x8 = x5; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x5); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x0 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x0 = _mm_xor_si128(x0, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x0 = _mm_xor_si128(x0, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = x0; + x8 = x0; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x1 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x1 = _mm_xor_si128(x1, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x1 = _mm_xor_si128(x1, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + /* H ^ 4 */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = x0; + x8 = x0; + x11 = _mm_clmulepi64_si128(x11, x0, 0x11); + x8 = _mm_clmulepi64_si128(x8, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x3 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x3 = _mm_xor_si128(x3, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x3 = _mm_xor_si128(x3, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x3); + /* H ^ 5 */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = x1; + x8 = x1; + x11 = _mm_clmulepi64_si128(x11, x0, 0x11); + x8 = _mm_clmulepi64_si128(x8, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = x1; + x8 = x1; + x11 = _mm_clmulepi64_si128(x11, x1, 0x11); + x8 = _mm_clmulepi64_si128(x8, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = x3; + x8 = x3; + x11 = _mm_clmulepi64_si128(x11, x1, 0x11); + x8 = _mm_clmulepi64_si128(x8, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x9 = _mm_shuffle_epi32(x3, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = x3; + x8 = x3; + x11 = _mm_clmulepi64_si128(x11, x3, 0x11); + x8 = _mm_clmulepi64_si128(x8, x3, 0); + x9 = _mm_xor_si128(x9, x3); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); +L_AES_GCM_decrypt_aesni_ghash_128: + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_bswap_epi64, 0)); + x0 = x8; + x8 = _mm_shuffle_epi8(x8, x1); + x9 = x0; + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = x0; + x10 = _mm_add_epi32(x10, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = x0; + x11 = _mm_add_epi32(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = x0; + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = x0; + x13 = _mm_add_epi32(x13, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = x0; + x14 = _mm_add_epi32(x14, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = x0; + x15 = _mm_add_epi32(x15, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x2); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x1 = _mm_xor_si128(x1, x7); + x5 = _mm_xor_si128(x5, x0); + x3 = x0; + x3 = _mm_clmulepi64_si128(x3, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x2 = x0; + x2 = _mm_clmulepi64_si128(x2, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x1 = _mm_clmulepi64_si128(x1, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x3); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x5 = x1; + x1 = _mm_bsrli_si128(x1, 8); + x5 = _mm_bslli_si128(x5, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x1); + x7 = x2; + x4 = x2; + x5 = x2; + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x7 = _mm_slli_epi32(x7, 31); + x4 = _mm_slli_epi32(x4, 30); + x5 = _mm_slli_epi32(x5, 25); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_xor_si128(x7, x5); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x4 = x7; + x7 = _mm_bslli_si128(x7, 12); + x4 = _mm_bsrli_si128(x4, 4); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x2 = _mm_xor_si128(x2, x7); + x5 = x2; + x1 = x2; + x0 = x2; + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_srli_epi32(x5, 1); + x1 = _mm_srli_epi32(x1, 2); + x0 = _mm_srli_epi32(x0, 7); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_xor_si128(x5, x1); + x5 = _mm_xor_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_xor_si128(x5, x4); + x2 = _mm_xor_si128(x2, x5); + x2 = _mm_xor_si128(x2, x3); + zf19 = (word32)r10; + zf20 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf21 = (word32)r10; + zf22 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_aesni_aesenc_128_ghash_avx_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + rbx = (word32)((word32)rbx + 0x80); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_aesni_ghash_128; + } + x6 = x2; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_decrypt_aesni_done_128: + rdx = (word32)((word32)r9); + if (((word32)rbx) >= ((word32)rdx)) { + goto L_AES_GCM_decrypt_aesni_done_dec; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_aesni_last_block_done; + } +L_AES_GCM_decrypt_aesni_last_block_start: + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = x5; + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x1 = _mm_xor_si128(x1, x6); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x9 = x8; + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x9); + x10 = x1; + x10 = _mm_clmulepi64_si128(x10, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x11 = x1; + x11 = _mm_clmulepi64_si128(x11, x0, 1); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x12 = x1; + x12 = _mm_clmulepi64_si128(x12, x0, 0); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x10 = _mm_xor_si128(x10, x11); + x2 = x10; + x10 = _mm_bsrli_si128(x10, 8); + x2 = _mm_bslli_si128(x2, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x3 = x1; + x2 = _mm_xor_si128(x2, x12); + x3 = _mm_xor_si128(x3, x10); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_mod2_128, 0)); + x11 = x2; + x11 = _mm_clmulepi64_si128(x11, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x10 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x11 = x10; + x11 = _mm_clmulepi64_si128(x11, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x6 = _mm_shuffle_epi32(x10, 0x4e); + x6 = _mm_xor_si128(x6, x11); + x6 = _mm_xor_si128(x6, x3); + zf23 = (word32)r10; + zf24 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_decrypt_aesni_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf25 = (word32)r10; + zf26 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_decrypt_aesni_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_aesni_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_aesni_last_block_start; + } +L_AES_GCM_decrypt_aesni_last_block_done: + rcx = (word32)((word32)r9); + rdx = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x4 = _mm_xor_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf27 = (word32)r10; + zf28 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x9); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf29 = (word32)r10; + zf30 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x9); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_aesenc_avx_last: + x4 = _mm_aesenclast_si128(x4, x9); + rsp = (word64)(rsp - 32); + rcx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + x0 = _mm_setzero_si128(); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); +L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + WC_S8(rsp, rcx + 16) = (byte)((byte)r13); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(rsi, rbx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_loop; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + rsp = (word64)(rsp + 32); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x4); + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = x6; + x8 = x6; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x6 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x6, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); +L_AES_GCM_decrypt_aesni_aesenc_last15_dec_avx_done: +L_AES_GCM_decrypt_aesni_done_dec: + rdx = (word32)((word32)r9); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_insert_epi64(x0, (long long)rdx, 0); + x0 = _mm_insert_epi64(x0, (long long)rcx, 1); + x6 = _mm_xor_si128(x6, x0); + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = x6; + x8 = x6; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x6 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x6, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + x0 = _mm_xor_si128(x0, x6); + if (((word32)r14) == (0x10)) { + goto L_AES_GCM_decrypt_aesni_cmp_tag_16; + } + rsp = (word64)(rsp - 16); + rcx = (word64)(0); + rbx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_aesni_cmp_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(r8, + rcx)) & 0xff); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)((byte)rbx | ( + byte)r13) & 0xff); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)r14)) { + goto L_AES_GCM_decrypt_aesni_cmp_tag_loop; + } + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)(((byte)rbx) == (0)) & 0xff); + rsp = (word64)(rsp + 16); + rcx = (word64)(0); + goto L_AES_GCM_decrypt_aesni_cmp_tag_done; +L_AES_GCM_decrypt_aesni_cmp_tag_16: + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x0 = _mm_cmpeq_epi8(x0, x1); + rdx = (word32)((word32)_mm_movemask_epi8(x0)); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + rbx = (word32)(0); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)(((word32)rdx) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_aesni_cmp_tag_done: + WC_S32(rbp, 0) = (word32)((word32)rbx); +} + +#ifdef WOLFSSL_AESGCM_STREAM +WC_X64I_TARGET("sse2,ssse3,sse4.1,aes,pclmul") +WOLFSSL_LOCAL void AES_GCM_init_aesni(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, rsp, rdx, rcx = 0, r13 = 0, + r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)ivec; + r11 = (word64)(word32)ibytes; + r8 = (word64)(size_t)h; + r9 = (word64)(size_t)counter; + rax = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_setzero_si128(); + rdx = (word32)((word32)r11); + if (((word32)rdx) != (0xc)) { + goto L_AES_GCM_init_aesni_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_insert_epi64(x4, (long long)WC_L64(r10, 0), 0); + x4 = _mm_insert_epi32(x4, (int)WC_L32(r10, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x1 = x4; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_xor_si128(x1, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + zf1 = (word32)rsi; + zf2 = 0xb; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_init_aesni_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + zf3 = (word32)rsi; + zf4 = 0xd; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_init_aesni_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_aesni_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x6); + x1 = _mm_aesenclast_si128(x1, x6); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x15 = x1; + goto L_AES_GCM_init_aesni_iv_done; +L_AES_GCM_init_aesni_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf5 = (word32)rsi; + zf6 = 0xb; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_init_aesni_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x8); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf7 = (word32)rsi; + zf8 = 0xd; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_init_aesni_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x8); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_aesni_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x8); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf9 = (word32)rdx; + zf10 = 0; + rcx = (word64)(0); + if ((zf9) == (zf10)) { + goto L_AES_GCM_init_aesni_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_init_aesni_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_init_aesni_calc_iv_16_loop: + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, rcx)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x6; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x6; + x1 = x6; + x2 = x6; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = x6; + x3 = x6; + x0 = x6; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_aesni_calc_iv_16_loop; + } + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_init_aesni_calc_iv_done; + } +L_AES_GCM_init_aesni_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x7 = _mm_setzero_si128(); + r13 = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x7); +L_AES_GCM_init_aesni_calc_iv_loop: + r12 = (word32)((word32)WC_L8(r10, rcx)); + WC_S8(rsp, r13) = (byte)((byte)r12); + rcx = (word32)((word32)rcx + 1); + r13 = (word32)((word32)r13 + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_aesni_calc_iv_loop; + } + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x6; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x6; + x1 = x6; + x2 = x6; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = x6; + x3 = x6; + x0 = x6; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); +L_AES_GCM_init_aesni_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_insert_epi64(x0, (long long)rdx, 0); + x4 = _mm_xor_si128(x4, x0); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x6; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x6; + x1 = x6; + x2 = x6; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = x6; + x3 = x6; + x0 = x6; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf11 = (word32)rsi; + zf12 = 0xb; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_init_aesni_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x8); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf13 = (word32)rsi; + zf14 = 0xd; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_init_aesni_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x8); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_aesni_calc_iv_2_aesenc_avx_last: + x7 = _mm_aesenclast_si128(x7, x8); + x15 = x7; +L_AES_GCM_init_aesni_iv_done: + _mm_storeu_si128((__m128i*)WC_PW(rax, 0), x15); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x4); +} + +WC_X64I_TARGET("sse2,ssse3,pclmul") +WOLFSSL_LOCAL void AES_GCM_aad_update_aesni(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5, x6, x7 = _mm_setzero_si128(); + + rdi = (word64)(size_t)addt; + rsi = (word64)(word32)abytes; + rdx = (word64)(size_t)tag; + rax = (word64)(size_t)h; + + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + rcx = (word32)(0); +L_AES_GCM_aad_update_aesni_16_loop: + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rcx)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x7); + x1 = _mm_shuffle_epi32(x5, 0x4e); + x2 = _mm_shuffle_epi32(x6, 0x4e); + x3 = x6; + x0 = x6; + x3 = _mm_clmulepi64_si128(x3, x5, 0x11); + x0 = _mm_clmulepi64_si128(x0, x5, 0); + x1 = _mm_xor_si128(x1, x5); + x2 = _mm_xor_si128(x2, x6); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = x0; + x5 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x5 = _mm_xor_si128(x5, x1); + x0 = x4; + x1 = x5; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x4 = _mm_slli_epi32(x4, 1); + x5 = _mm_slli_epi32(x5, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x5 = _mm_or_si128(x5, x2); + x4 = _mm_or_si128(x4, x0); + x5 = _mm_or_si128(x5, x1); + x0 = x4; + x1 = x4; + x2 = x4; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x4 = _mm_xor_si128(x4, x0); + x2 = x4; + x3 = x4; + x0 = x4; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x4); + x5 = _mm_xor_si128(x5, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rsi)) { + goto L_AES_GCM_aad_update_aesni_16_loop; + } + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x5); +} + +WC_X64I_TARGET("sse2,ssse3,aes") +WOLFSSL_LOCAL void AES_GCM_encrypt_block_aesni(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8; + __m128i x0, x1; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(size_t)counter; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x1 = x0; + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x1); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf1 = (word32)rsi; + zf2 = 0xb; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_block_aesni_aesenc_block_aesenc_avx_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf3 = (word32)rsi; + zf4 = 0xd; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_block_aesni_aesenc_block_aesenc_avx_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_block_aesni_aesenc_block_aesenc_avx_last: + x0 = _mm_aesenclast_si128(x0, x1); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); +} + +WC_X64I_TARGET("sse2,ssse3,pclmul") +WOLFSSL_LOCAL void AES_GCM_ghash_block_aesni(const unsigned char* addt, + unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, rdx; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + + rdi = (word64)(size_t)addt; + rsi = (word64)(size_t)tag; + rdx = (word64)(size_t)h; + + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = x5; + x0 = x5; + x3 = _mm_clmulepi64_si128(x3, x4, 0x11); + x0 = _mm_clmulepi64_si128(x0, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = x6; + x1 = x4; + x0 = _mm_srli_epi32(x0, 31); + x1 = _mm_srli_epi32(x1, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = x0; + x0 = _mm_bslli_si128(x0, 4); + x2 = _mm_bsrli_si128(x2, 12); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = x6; + x1 = x6; + x2 = x6; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = x6; + x3 = x6; + x0 = x6; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x4); +} + +WC_X64I_TARGET("sse2,ssse3,aes,pclmul") +WOLFSSL_LOCAL void AES_GCM_encrypt_update_aesni(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, r13, rcx = 0, + rdx = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5, x6, x7 = _mm_setzero_si128(), x8, + x9, x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[20]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 160); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x9 = x5; + x8 = x5; + x9 = _mm_srli_epi64(x9, 63); + x8 = _mm_slli_epi64(x8, 1); + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_or_si128(x8, x9); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + r14 = (word64)(0); + zf1 = (word32)r8; + zf2 = 0x80; + r13 = (word32)((word32)r8); + if ((zf1) < (zf2)) { + goto L_AES_GCM_encrypt_update_aesni_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + x2 = x6; + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x5, 0x4e); + x11 = x5; + x8 = x5; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x5); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x0 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x0 = _mm_xor_si128(x0, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x0 = _mm_xor_si128(x0, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = x0; + x8 = x0; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x1 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x1 = _mm_xor_si128(x1, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x1 = _mm_xor_si128(x1, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + /* H ^ 4 */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = x0; + x8 = x0; + x11 = _mm_clmulepi64_si128(x11, x0, 0x11); + x8 = _mm_clmulepi64_si128(x8, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x3 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x3 = _mm_xor_si128(x3, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x3 = _mm_xor_si128(x3, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x3); + /* H ^ 5 */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = x1; + x8 = x1; + x11 = _mm_clmulepi64_si128(x11, x0, 0x11); + x8 = _mm_clmulepi64_si128(x8, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = x1; + x8 = x1; + x11 = _mm_clmulepi64_si128(x11, x1, 0x11); + x8 = _mm_clmulepi64_si128(x8, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = x3; + x8 = x3; + x11 = _mm_clmulepi64_si128(x11, x1, 0x11); + x8 = _mm_clmulepi64_si128(x8, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x9 = _mm_shuffle_epi32(x3, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = x3; + x8 = x3; + x11 = _mm_clmulepi64_si128(x11, x3, 0x11); + x8 = _mm_clmulepi64_si128(x8, x3, 0); + x9 = _mm_xor_si128(x9, x3); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + /* First 128 bytes of input */ + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_bswap_epi64, 0)); + x0 = x8; + x8 = _mm_shuffle_epi8(x8, x1); + x9 = x0; + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = x0; + x10 = _mm_add_epi32(x10, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = x0; + x11 = _mm_add_epi32(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = x0; + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = x0; + x13 = _mm_add_epi32(x13, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = x0; + x14 = _mm_add_epi32(x14, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = x0; + x15 = _mm_add_epi32(x15, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf3 = (word32)rsi; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_update_aesni_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf5 = (word32)rsi; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_encrypt_update_aesni_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_aesni_enc_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(r10, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(r10, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(r10, 112), x15); + zf7 = (word32)r13; + zf8 = 0x80; + r14 = (word32)(0x80); + if ((zf7) <= (zf8)) { + goto L_AES_GCM_encrypt_update_aesni_end_128; + } + /* More 128 bytes of input */ +L_AES_GCM_encrypt_update_aesni_ghash_128: + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_bswap_epi64, 0)); + x0 = x8; + x8 = _mm_shuffle_epi8(x8, x1); + x9 = x0; + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = x0; + x10 = _mm_add_epi32(x10, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = x0; + x11 = _mm_add_epi32(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = x0; + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = x0; + x13 = _mm_add_epi32(x13, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = x0; + x14 = _mm_add_epi32(x14, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = x0; + x15 = _mm_add_epi32(x15, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -128)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x2); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x1 = _mm_xor_si128(x1, x7); + x5 = _mm_xor_si128(x5, x0); + x3 = x0; + x3 = _mm_clmulepi64_si128(x3, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x2 = x0; + x2 = _mm_clmulepi64_si128(x2, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x1 = _mm_clmulepi64_si128(x1, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x3); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -112)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -96)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -80)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -64)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -48)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -32)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -16)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x5 = x1; + x1 = _mm_bsrli_si128(x1, 8); + x5 = _mm_bslli_si128(x5, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x1); + x7 = x2; + x4 = x2; + x5 = x2; + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x7 = _mm_slli_epi32(x7, 31); + x4 = _mm_slli_epi32(x4, 30); + x5 = _mm_slli_epi32(x5, 25); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_xor_si128(x7, x5); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x4 = x7; + x7 = _mm_bslli_si128(x7, 12); + x4 = _mm_bsrli_si128(x4, 4); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x2 = _mm_xor_si128(x2, x7); + x5 = x2; + x1 = x2; + x0 = x2; + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_srli_epi32(x5, 1); + x1 = _mm_srli_epi32(x1, 2); + x0 = _mm_srli_epi32(x0, 7); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_xor_si128(x5, x1); + x5 = _mm_xor_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_xor_si128(x5, x4); + x2 = _mm_xor_si128(x2, x5); + x2 = _mm_xor_si128(x2, x3); + zf9 = (word32)rsi; + zf10 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_encrypt_update_aesni_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf11 = (word32)rsi; + zf12 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_encrypt_update_aesni_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_aesni_aesenc_128_ghash_avx_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + r14 = (word32)((word32)r14 + 0x80); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_aesni_ghash_128; + } +L_AES_GCM_encrypt_update_aesni_end_128: + x4 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_bswap_mask, 0)); + x8 = _mm_shuffle_epi8(x8, x4); + x9 = _mm_shuffle_epi8(x9, x4); + x10 = _mm_shuffle_epi8(x10, x4); + x11 = _mm_shuffle_epi8(x11, x4); + x8 = _mm_xor_si128(x8, x2); + x12 = _mm_shuffle_epi8(x12, x4); + x13 = _mm_shuffle_epi8(x13, x4); + x14 = _mm_shuffle_epi8(x14, x4); + x15 = _mm_shuffle_epi8(x15, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x1 = _mm_shuffle_epi32(x8, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x8, 0x11); + x0 = _mm_clmulepi64_si128(x0, x8, 0); + x1 = _mm_xor_si128(x1, x8); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x1 = _mm_shuffle_epi32(x9, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x9, 0x11); + x0 = _mm_clmulepi64_si128(x0, x9, 0); + x1 = _mm_xor_si128(x1, x9); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x1 = _mm_shuffle_epi32(x10, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x10, 0x11); + x0 = _mm_clmulepi64_si128(x0, x10, 0); + x1 = _mm_xor_si128(x1, x10); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x1 = _mm_shuffle_epi32(x11, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x11, 0x11); + x0 = _mm_clmulepi64_si128(x0, x11, 0); + x1 = _mm_xor_si128(x1, x11); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x1 = _mm_shuffle_epi32(x12, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x12, 0x11); + x0 = _mm_clmulepi64_si128(x0, x12, 0); + x1 = _mm_xor_si128(x1, x12); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x1 = _mm_shuffle_epi32(x13, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x13, 0x11); + x0 = _mm_clmulepi64_si128(x0, x13, 0); + x1 = _mm_xor_si128(x1, x13); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x1 = _mm_shuffle_epi32(x14, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x14, 0x11); + x0 = _mm_clmulepi64_si128(x0, x14, 0); + x1 = _mm_xor_si128(x1, x14); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x1 = _mm_shuffle_epi32(x15, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = x7; + x0 = x7; + x3 = _mm_clmulepi64_si128(x3, x15, 0x11); + x0 = _mm_clmulepi64_si128(x0, x15, 0); + x1 = _mm_xor_si128(x1, x15); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x2 = x1; + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x2, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = x4; + x1 = x4; + x2 = x4; + x0 = _mm_slli_epi32(x0, 31); + x1 = _mm_slli_epi32(x1, 30); + x2 = _mm_slli_epi32(x2, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x4 = _mm_xor_si128(x4, x0); + x2 = x4; + x3 = x4; + x0 = x4; + x2 = _mm_srli_epi32(x2, 1); + x3 = _mm_srli_epi32(x3, 2); + x0 = _mm_srli_epi32(x0, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x4); + x6 = _mm_xor_si128(x6, x2); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_encrypt_update_aesni_done_128: + rdx = (word32)((word32)r8); + if (((word32)r14) >= ((word32)rdx)) { + goto L_AES_GCM_encrypt_update_aesni_done_enc; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_aesni_last_block_done; + } + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x9 = x8; + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf13 = (word32)rsi; + zf14 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_update_aesni_aesenc_block_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf15 = (word32)rsi; + zf16 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_encrypt_update_aesni_aesenc_block_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_aesni_aesenc_block_aesenc_avx_last: + x8 = _mm_aesenclast_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_aesni_last_block_ghash; + } +L_AES_GCM_encrypt_update_aesni_last_block_start: + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x9 = x8; + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x9); + x10 = x6; + x10 = _mm_clmulepi64_si128(x10, x5, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x11 = x6; + x11 = _mm_clmulepi64_si128(x11, x5, 1); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x12 = x6; + x12 = _mm_clmulepi64_si128(x12, x5, 0); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x1 = x6; + x1 = _mm_clmulepi64_si128(x1, x5, 0x11); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x10 = _mm_xor_si128(x10, x11); + x2 = x10; + x10 = _mm_bsrli_si128(x10, 8); + x2 = _mm_bslli_si128(x2, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x3 = x1; + x2 = _mm_xor_si128(x2, x12); + x3 = _mm_xor_si128(x3, x10); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_mod2_128, 0)); + x11 = x2; + x11 = _mm_clmulepi64_si128(x11, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x10 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x11 = x10; + x11 = _mm_clmulepi64_si128(x11, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x6 = _mm_shuffle_epi32(x10, 0x4e); + x6 = _mm_xor_si128(x6, x11); + x6 = _mm_xor_si128(x6, x3); + zf17 = (word32)rsi; + zf18 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_encrypt_update_aesni_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf19 = (word32)rsi; + zf20 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_encrypt_update_aesni_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_aesni_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_aesni_last_block_start; + } +L_AES_GCM_encrypt_update_aesni_last_block_ghash: + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = x6; + x8 = x6; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x6 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x6, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); +L_AES_GCM_encrypt_update_aesni_last_block_done: +L_AES_GCM_encrypt_update_aesni_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x6); +} + +WC_X64I_TARGET("sse2,ssse3,sse4.1,pclmul") +WOLFSSL_LOCAL void AES_GCM_encrypt_final_aesni(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr) +{ + word64 rdi, rsi, rax, r10, r11, r9, r8, rsp, rdx, rcx, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x4, x5, x6, x7, x8, x9, x10, x11, x12, + x13; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rax = (word64)(word32)tbytes; + r10 = (word64)(word32)nbytes; + r11 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + r8 = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = x5; + x7 = x5; + x8 = _mm_srli_epi64(x8, 63); + x7 = _mm_slli_epi64(x7, 1); + x8 = _mm_bslli_si128(x8, 8); + x7 = _mm_or_si128(x7, x8); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x7); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_insert_epi64(x0, (long long)rdx, 0); + x0 = _mm_insert_epi64(x0, (long long)rcx, 1); + x4 = _mm_xor_si128(x4, x0); + x8 = _mm_shuffle_epi32(x5, 0x4e); + x9 = _mm_shuffle_epi32(x4, 0x4e); + x10 = x4; + x7 = x4; + x10 = _mm_clmulepi64_si128(x10, x5, 0x11); + x7 = _mm_clmulepi64_si128(x7, x5, 0); + x8 = _mm_xor_si128(x8, x5); + x9 = _mm_xor_si128(x9, x4); + x8 = _mm_clmulepi64_si128(x8, x9, 0); + x8 = _mm_xor_si128(x8, x7); + x8 = _mm_xor_si128(x8, x10); + x9 = x8; + x4 = x10; + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_bsrli_si128(x8, 8); + x7 = _mm_xor_si128(x7, x9); + x4 = _mm_xor_si128(x4, x8); + x11 = x7; + x12 = x7; + x13 = x7; + x11 = _mm_slli_epi32(x11, 31); + x12 = _mm_slli_epi32(x12, 30); + x13 = _mm_slli_epi32(x13, 25); + x11 = _mm_xor_si128(x11, x12); + x11 = _mm_xor_si128(x11, x13); + x12 = x11; + x12 = _mm_bsrli_si128(x12, 4); + x11 = _mm_bslli_si128(x11, 12); + x7 = _mm_xor_si128(x7, x11); + x13 = x7; + x9 = x7; + x8 = x7; + x13 = _mm_srli_epi32(x13, 1); + x9 = _mm_srli_epi32(x9, 2); + x8 = _mm_srli_epi32(x8, 7); + x13 = _mm_xor_si128(x13, x9); + x13 = _mm_xor_si128(x13, x8); + x13 = _mm_xor_si128(x13, x12); + x13 = _mm_xor_si128(x13, x7); + x4 = _mm_xor_si128(x4, x13); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x0 = x6; + x0 = _mm_xor_si128(x0, x4); + if (((word32)rax) == (0x10)) { + goto L_AES_GCM_encrypt_final_aesni_store_tag_16; + } + rcx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_final_aesni_store_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + WC_S8(rsi, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)rax)) { + goto L_AES_GCM_encrypt_final_aesni_store_tag_loop; + } + goto L_AES_GCM_encrypt_final_aesni_store_tag_done; +L_AES_GCM_encrypt_final_aesni_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); +L_AES_GCM_encrypt_final_aesni_store_tag_done: + ; +} + +WC_X64I_TARGET("sse2,ssse3,aes,pclmul") +WOLFSSL_LOCAL void AES_GCM_decrypt_update_aesni(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, r13, rcx = 0, + rdx = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5, x6, x7 = _mm_setzero_si128(), x8, + x9, x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[24]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 168); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x9 = x5; + x8 = x5; + x9 = _mm_srli_epi64(x9, 63); + x8 = _mm_slli_epi64(x8, 1); + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_or_si128(x8, x9); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + r14 = (word32)(0); + zf1 = (word32)r8; + zf2 = 0x80; + r13 = (word32)((word32)r8); + if ((zf1) < (zf2)) { + goto L_AES_GCM_decrypt_update_aesni_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + x2 = x6; + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x5, 0x4e); + x11 = x5; + x8 = x5; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x5); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x0 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x0 = _mm_xor_si128(x0, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x0 = _mm_xor_si128(x0, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = x0; + x8 = x0; + x11 = _mm_clmulepi64_si128(x11, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x1 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x1 = _mm_xor_si128(x1, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x1 = _mm_xor_si128(x1, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + /* H ^ 4 */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = x0; + x8 = x0; + x11 = _mm_clmulepi64_si128(x11, x0, 0x11); + x8 = _mm_clmulepi64_si128(x8, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x3 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x3 = _mm_xor_si128(x3, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x3 = _mm_xor_si128(x3, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x3); + /* H ^ 5 */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = x1; + x8 = x1; + x11 = _mm_clmulepi64_si128(x11, x0, 0x11); + x8 = _mm_clmulepi64_si128(x8, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = x1; + x8 = x1; + x11 = _mm_clmulepi64_si128(x11, x1, 0x11); + x8 = _mm_clmulepi64_si128(x8, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = x3; + x8 = x3; + x11 = _mm_clmulepi64_si128(x11, x1, 0x11); + x8 = _mm_clmulepi64_si128(x8, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x9 = _mm_shuffle_epi32(x3, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = x3; + x8 = x3; + x11 = _mm_clmulepi64_si128(x11, x3, 0x11); + x8 = _mm_clmulepi64_si128(x8, x3, 0); + x9 = _mm_xor_si128(x9, x3); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = x9; + x7 = x11; + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x7, x9); + x12 = x8; + x13 = x8; + x14 = x8; + x12 = _mm_slli_epi32(x12, 31); + x13 = _mm_slli_epi32(x13, 30); + x14 = _mm_slli_epi32(x14, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = x12; + x13 = _mm_bsrli_si128(x13, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = x8; + x10 = x8; + x9 = x8; + x14 = _mm_srli_epi32(x14, 1); + x10 = _mm_srli_epi32(x10, 2); + x9 = _mm_srli_epi32(x9, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); +L_AES_GCM_decrypt_update_aesni_ghash_128: + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_bswap_epi64, 0)); + x0 = x8; + x8 = _mm_shuffle_epi8(x8, x1); + x9 = x0; + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = x0; + x10 = _mm_add_epi32(x10, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = x0; + x11 = _mm_add_epi32(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = x0; + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = x0; + x13 = _mm_add_epi32(x13, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = x0; + x14 = _mm_add_epi32(x14, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = x0; + x15 = _mm_add_epi32(x15, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x2); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x1 = _mm_xor_si128(x1, x7); + x5 = _mm_xor_si128(x5, x0); + x3 = x0; + x3 = _mm_clmulepi64_si128(x3, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x2 = x0; + x2 = _mm_clmulepi64_si128(x2, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x1 = _mm_clmulepi64_si128(x1, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x3); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = x0; + x6 = _mm_clmulepi64_si128(x6, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x7 = _mm_clmulepi64_si128(x7, x0, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x5 = x1; + x1 = _mm_bsrli_si128(x1, 8); + x5 = _mm_bslli_si128(x5, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x1); + x7 = x2; + x4 = x2; + x5 = x2; + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x7 = _mm_slli_epi32(x7, 31); + x4 = _mm_slli_epi32(x4, 30); + x5 = _mm_slli_epi32(x5, 25); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_xor_si128(x7, x5); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x4 = x7; + x7 = _mm_bslli_si128(x7, 12); + x4 = _mm_bsrli_si128(x4, 4); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x2 = _mm_xor_si128(x2, x7); + x5 = x2; + x1 = x2; + x0 = x2; + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_srli_epi32(x5, 1); + x1 = _mm_srli_epi32(x1, 2); + x0 = _mm_srli_epi32(x0, 7); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_xor_si128(x5, x1); + x5 = _mm_xor_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_xor_si128(x5, x4); + x2 = _mm_xor_si128(x2, x5); + x2 = _mm_xor_si128(x2, x3); + zf3 = (word32)rsi; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_update_aesni_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf5 = (word32)rsi; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_decrypt_update_aesni_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_decrypt_update_aesni_aesenc_128_ghash_avx_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + r14 = (word32)((word32)r14 + 0x80); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_aesni_ghash_128; + } + x6 = x2; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_decrypt_update_aesni_done_128: + rdx = (word32)((word32)r8); + if (((word32)r14) >= ((word32)rdx)) { + goto L_AES_GCM_decrypt_update_aesni_done_dec; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_update_aesni_last_block_done; + } +L_AES_GCM_decrypt_update_aesni_last_block_start: + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = x5; + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x1 = _mm_xor_si128(x1, x6); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x9 = x8; + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_one, + 0))); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x9); + x10 = x1; + x10 = _mm_clmulepi64_si128(x10, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x11 = x1; + x11 = _mm_clmulepi64_si128(x11, x0, 1); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x12 = x1; + x12 = _mm_clmulepi64_si128(x12, x0, 0); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x10 = _mm_xor_si128(x10, x11); + x2 = x10; + x10 = _mm_bsrli_si128(x10, 8); + x2 = _mm_bslli_si128(x2, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x3 = x1; + x2 = _mm_xor_si128(x2, x12); + x3 = _mm_xor_si128(x3, x10); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcm_mod2_128, 0)); + x11 = x2; + x11 = _mm_clmulepi64_si128(x11, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x10 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x11 = x10; + x11 = _mm_clmulepi64_si128(x11, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x6 = _mm_shuffle_epi32(x10, 0x4e); + x6 = _mm_xor_si128(x6, x11); + x6 = _mm_xor_si128(x6, x3); + zf7 = (word32)rsi; + zf8 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_decrypt_update_aesni_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf9 = (word32)rsi; + zf10 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_decrypt_update_aesni_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_decrypt_update_aesni_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_aesni_last_block_start; + } +L_AES_GCM_decrypt_update_aesni_last_block_done: +L_AES_GCM_decrypt_update_aesni_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x6); +} + +WC_X64I_TARGET("sse2,ssse3,sse4.1,pclmul") +WOLFSSL_LOCAL void AES_GCM_decrypt_final_aesni(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res) +{ + word64 rdi, rsi, rax, r10, r11, r9, r8, rbp, rsp, rdx, rcx, r12 = 0, + r13 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), x5, x6, x7, + x8, x9, x10, x11, x12, x13, x15; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rax = (word64)(word32)tbytes; + r10 = (word64)(word32)nbytes; + r11 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + r8 = (word64)(size_t)initCtr; + rbp = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = x5; + x7 = x5; + x8 = _mm_srli_epi64(x8, 63); + x7 = _mm_slli_epi64(x7, 1); + x8 = _mm_bslli_si128(x8, 8); + x7 = _mm_or_si128(x7, x8); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x7); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_insert_epi64(x0, (long long)rdx, 0); + x0 = _mm_insert_epi64(x0, (long long)rcx, 1); + x6 = _mm_xor_si128(x6, x0); + x8 = _mm_shuffle_epi32(x5, 0x4e); + x9 = _mm_shuffle_epi32(x6, 0x4e); + x10 = x6; + x7 = x6; + x10 = _mm_clmulepi64_si128(x10, x5, 0x11); + x7 = _mm_clmulepi64_si128(x7, x5, 0); + x8 = _mm_xor_si128(x8, x5); + x9 = _mm_xor_si128(x9, x6); + x8 = _mm_clmulepi64_si128(x8, x9, 0); + x8 = _mm_xor_si128(x8, x7); + x8 = _mm_xor_si128(x8, x10); + x9 = x8; + x6 = x10; + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_bsrli_si128(x8, 8); + x7 = _mm_xor_si128(x7, x9); + x6 = _mm_xor_si128(x6, x8); + x11 = x7; + x12 = x7; + x13 = x7; + x11 = _mm_slli_epi32(x11, 31); + x12 = _mm_slli_epi32(x12, 30); + x13 = _mm_slli_epi32(x13, 25); + x11 = _mm_xor_si128(x11, x12); + x11 = _mm_xor_si128(x11, x13); + x12 = x11; + x12 = _mm_bsrli_si128(x12, 4); + x11 = _mm_bslli_si128(x11, 12); + x7 = _mm_xor_si128(x7, x11); + x13 = x7; + x9 = x7; + x8 = x7; + x13 = _mm_srli_epi32(x13, 1); + x9 = _mm_srli_epi32(x9, 2); + x8 = _mm_srli_epi32(x8, 7); + x13 = _mm_xor_si128(x13, x9); + x13 = _mm_xor_si128(x13, x8); + x13 = _mm_xor_si128(x13, x12); + x13 = _mm_xor_si128(x13, x7); + x6 = _mm_xor_si128(x6, x13); + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_bswap_mask, 0))); + x0 = x15; + x0 = _mm_xor_si128(x0, x6); + if (((word32)rax) == (0x10)) { + goto L_AES_GCM_decrypt_final_aesni_cmp_tag_16; + } + rsp = (word64)(rsp - 16); + rcx = (word64)(0); + r12 = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_final_aesni_cmp_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsi, + rcx)) & 0xff); + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)((byte)r12 | ( + byte)r13) & 0xff); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)rax)) { + goto L_AES_GCM_decrypt_final_aesni_cmp_tag_loop; + } + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)(((byte)r12) == (0)) & 0xff); + rsp = (word64)(rsp + 16); + rcx = (word64)(0); + goto L_AES_GCM_decrypt_final_aesni_cmp_tag_done; +L_AES_GCM_decrypt_final_aesni_cmp_tag_16: + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_cmpeq_epi8(x0, x1); + rdx = (word32)((word32)_mm_movemask_epi8(x0)); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + r12 = (word32)(0); + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)(((word32)rdx) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_final_aesni_cmp_tag_done: + WC_S32(rbp, 0) = (word32)((word32)r12); +} + +#endif /* WOLFSSL_AESGCM_STREAM */ +#ifdef WOLFSSL_AESGCM_SIV +XALIGNED(16) static const word64 L_aes_gcm_siv_bswap_mask[] WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +WC_X64I_TARGET("sse2,ssse3,pclmul") +WOLFSSL_LOCAL void AES_GCMSIV_polyval_aesni(unsigned char* s, + const unsigned char* h, const unsigned char* data, unsigned int blocks) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10 = 0; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)h; + rdx = (word64)(size_t)data; + rcx = (word64)(word32)blocks; + + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_siv_bswap_mask, 0))); + x9 = x1; + x9 = _mm_clmulepi64_si128(x9, x1, 0); + x10 = x1; + x10 = _mm_clmulepi64_si128(x10, x1, 0x11); + x13 = x1; + x13 = _mm_clmulepi64_si128(x13, x1, 0x10); + x14 = x1; + x14 = _mm_clmulepi64_si128(x14, x1, 1); + x13 = _mm_xor_si128(x13, x14); + x14 = x13; + x14 = _mm_bslli_si128(x14, 8); + x13 = _mm_bsrli_si128(x13, 8); + x9 = _mm_xor_si128(x9, x14); + x10 = _mm_xor_si128(x10, x13); + x5 = x9; + x5 = _mm_srli_epi32(x5, 31); + x6 = x10; + x6 = _mm_srli_epi32(x6, 31); + x9 = _mm_slli_epi32(x9, 1); + x10 = _mm_slli_epi32(x10, 1); + x7 = x5; + x7 = _mm_bsrli_si128(x7, 12); + x6 = _mm_bslli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 4); + x9 = _mm_or_si128(x9, x5); + x10 = _mm_or_si128(x10, x6); + x10 = _mm_or_si128(x10, x7); + x5 = x9; + x5 = _mm_slli_epi32(x5, 31); + x6 = x9; + x6 = _mm_slli_epi32(x6, 30); + x7 = x9; + x7 = _mm_slli_epi32(x7, 25); + x5 = _mm_xor_si128(x5, x6); + x5 = _mm_xor_si128(x5, x7); + x6 = x5; + x6 = _mm_bsrli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 12); + x9 = _mm_xor_si128(x9, x5); + x8 = x9; + x8 = _mm_srli_epi32(x8, 1); + x11 = x9; + x11 = _mm_srli_epi32(x11, 2); + x12 = x9; + x12 = _mm_srli_epi32(x12, 7); + x8 = _mm_xor_si128(x8, x11); + x8 = _mm_xor_si128(x8, x12); + x8 = _mm_xor_si128(x8, x6); + x9 = _mm_xor_si128(x9, x8); + x10 = _mm_xor_si128(x10, x9); + x2 = x10; + x9 = x2; + x9 = _mm_clmulepi64_si128(x9, x1, 0); + x10 = x2; + x10 = _mm_clmulepi64_si128(x10, x1, 0x11); + x13 = x2; + x13 = _mm_clmulepi64_si128(x13, x1, 0x10); + x14 = x2; + x14 = _mm_clmulepi64_si128(x14, x1, 1); + x13 = _mm_xor_si128(x13, x14); + x14 = x13; + x14 = _mm_bslli_si128(x14, 8); + x13 = _mm_bsrli_si128(x13, 8); + x9 = _mm_xor_si128(x9, x14); + x10 = _mm_xor_si128(x10, x13); + x5 = x9; + x5 = _mm_srli_epi32(x5, 31); + x6 = x10; + x6 = _mm_srli_epi32(x6, 31); + x9 = _mm_slli_epi32(x9, 1); + x10 = _mm_slli_epi32(x10, 1); + x7 = x5; + x7 = _mm_bsrli_si128(x7, 12); + x6 = _mm_bslli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 4); + x9 = _mm_or_si128(x9, x5); + x10 = _mm_or_si128(x10, x6); + x10 = _mm_or_si128(x10, x7); + x5 = x9; + x5 = _mm_slli_epi32(x5, 31); + x6 = x9; + x6 = _mm_slli_epi32(x6, 30); + x7 = x9; + x7 = _mm_slli_epi32(x7, 25); + x5 = _mm_xor_si128(x5, x6); + x5 = _mm_xor_si128(x5, x7); + x6 = x5; + x6 = _mm_bsrli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 12); + x9 = _mm_xor_si128(x9, x5); + x8 = x9; + x8 = _mm_srli_epi32(x8, 1); + x11 = x9; + x11 = _mm_srli_epi32(x11, 2); + x12 = x9; + x12 = _mm_srli_epi32(x12, 7); + x8 = _mm_xor_si128(x8, x11); + x8 = _mm_xor_si128(x8, x12); + x8 = _mm_xor_si128(x8, x6); + x9 = _mm_xor_si128(x9, x8); + x10 = _mm_xor_si128(x10, x9); + x3 = x10; + x9 = x2; + x9 = _mm_clmulepi64_si128(x9, x2, 0); + x10 = x2; + x10 = _mm_clmulepi64_si128(x10, x2, 0x11); + x13 = x2; + x13 = _mm_clmulepi64_si128(x13, x2, 0x10); + x14 = x2; + x14 = _mm_clmulepi64_si128(x14, x2, 1); + x13 = _mm_xor_si128(x13, x14); + x14 = x13; + x14 = _mm_bslli_si128(x14, 8); + x13 = _mm_bsrli_si128(x13, 8); + x9 = _mm_xor_si128(x9, x14); + x10 = _mm_xor_si128(x10, x13); + x5 = x9; + x5 = _mm_srli_epi32(x5, 31); + x6 = x10; + x6 = _mm_srli_epi32(x6, 31); + x9 = _mm_slli_epi32(x9, 1); + x10 = _mm_slli_epi32(x10, 1); + x7 = x5; + x7 = _mm_bsrli_si128(x7, 12); + x6 = _mm_bslli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 4); + x9 = _mm_or_si128(x9, x5); + x10 = _mm_or_si128(x10, x6); + x10 = _mm_or_si128(x10, x7); + x5 = x9; + x5 = _mm_slli_epi32(x5, 31); + x6 = x9; + x6 = _mm_slli_epi32(x6, 30); + x7 = x9; + x7 = _mm_slli_epi32(x7, 25); + x5 = _mm_xor_si128(x5, x6); + x5 = _mm_xor_si128(x5, x7); + x6 = x5; + x6 = _mm_bsrli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 12); + x9 = _mm_xor_si128(x9, x5); + x8 = x9; + x8 = _mm_srli_epi32(x8, 1); + x11 = x9; + x11 = _mm_srli_epi32(x11, 2); + x12 = x9; + x12 = _mm_srli_epi32(x12, 7); + x8 = _mm_xor_si128(x8, x11); + x8 = _mm_xor_si128(x8, x12); + x8 = _mm_xor_si128(x8, x6); + x9 = _mm_xor_si128(x9, x8); + x10 = _mm_xor_si128(x10, x9); + x4 = x10; + r8 = (word32)((word32)rcx); + r8 = (word32)((word32)r8 << 4); + r9 = (word32)((word32)r8); + r9 = (word32)((word32)r9 & 0xffffffc0); + rax = (word32)(0); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_GCMSIV_polyval_aesni_four_done; + } +L_AES_GCMSIV_polyval_aesni_four: + r10 = (word64)(rdx + rax); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x5 = _mm_xor_si128(x5, x0); + x9 = x5; + x9 = _mm_clmulepi64_si128(x9, x4, 0); + x10 = x5; + x10 = _mm_clmulepi64_si128(x10, x4, 0x11); + x13 = x5; + x13 = _mm_clmulepi64_si128(x13, x4, 0x10); + x14 = x5; + x14 = _mm_clmulepi64_si128(x14, x4, 1); + x13 = _mm_xor_si128(x13, x14); + x14 = x13; + x14 = _mm_bslli_si128(x14, 8); + x13 = _mm_bsrli_si128(x13, 8); + x9 = _mm_xor_si128(x9, x14); + x10 = _mm_xor_si128(x10, x13); + x11 = x6; + x11 = _mm_clmulepi64_si128(x11, x3, 0); + x12 = x6; + x12 = _mm_clmulepi64_si128(x12, x3, 0x11); + x13 = x6; + x13 = _mm_clmulepi64_si128(x13, x3, 0x10); + x14 = x6; + x14 = _mm_clmulepi64_si128(x14, x3, 1); + x13 = _mm_xor_si128(x13, x14); + x14 = x13; + x14 = _mm_bslli_si128(x14, 8); + x13 = _mm_bsrli_si128(x13, 8); + x11 = _mm_xor_si128(x11, x14); + x12 = _mm_xor_si128(x12, x13); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_xor_si128(x10, x12); + x11 = x7; + x11 = _mm_clmulepi64_si128(x11, x2, 0); + x12 = x7; + x12 = _mm_clmulepi64_si128(x12, x2, 0x11); + x13 = x7; + x13 = _mm_clmulepi64_si128(x13, x2, 0x10); + x14 = x7; + x14 = _mm_clmulepi64_si128(x14, x2, 1); + x13 = _mm_xor_si128(x13, x14); + x14 = x13; + x14 = _mm_bslli_si128(x14, 8); + x13 = _mm_bsrli_si128(x13, 8); + x11 = _mm_xor_si128(x11, x14); + x12 = _mm_xor_si128(x12, x13); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_xor_si128(x10, x12); + x11 = x8; + x11 = _mm_clmulepi64_si128(x11, x1, 0); + x12 = x8; + x12 = _mm_clmulepi64_si128(x12, x1, 0x11); + x13 = x8; + x13 = _mm_clmulepi64_si128(x13, x1, 0x10); + x14 = x8; + x14 = _mm_clmulepi64_si128(x14, x1, 1); + x13 = _mm_xor_si128(x13, x14); + x14 = x13; + x14 = _mm_bslli_si128(x14, 8); + x13 = _mm_bsrli_si128(x13, 8); + x11 = _mm_xor_si128(x11, x14); + x12 = _mm_xor_si128(x12, x13); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_xor_si128(x10, x12); + x5 = x9; + x5 = _mm_srli_epi32(x5, 31); + x6 = x10; + x6 = _mm_srli_epi32(x6, 31); + x9 = _mm_slli_epi32(x9, 1); + x10 = _mm_slli_epi32(x10, 1); + x7 = x5; + x7 = _mm_bsrli_si128(x7, 12); + x6 = _mm_bslli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 4); + x9 = _mm_or_si128(x9, x5); + x10 = _mm_or_si128(x10, x6); + x10 = _mm_or_si128(x10, x7); + x5 = x9; + x5 = _mm_slli_epi32(x5, 31); + x6 = x9; + x6 = _mm_slli_epi32(x6, 30); + x7 = x9; + x7 = _mm_slli_epi32(x7, 25); + x5 = _mm_xor_si128(x5, x6); + x5 = _mm_xor_si128(x5, x7); + x6 = x5; + x6 = _mm_bsrli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 12); + x9 = _mm_xor_si128(x9, x5); + x8 = x9; + x8 = _mm_srli_epi32(x8, 1); + x11 = x9; + x11 = _mm_srli_epi32(x11, 2); + x12 = x9; + x12 = _mm_srli_epi32(x12, 7); + x8 = _mm_xor_si128(x8, x11); + x8 = _mm_xor_si128(x8, x12); + x8 = _mm_xor_si128(x8, x6); + x9 = _mm_xor_si128(x9, x8); + x10 = _mm_xor_si128(x10, x9); + x0 = x10; + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_GCMSIV_polyval_aesni_four; + } +L_AES_GCMSIV_polyval_aesni_four_done: +L_AES_GCMSIV_polyval_aesni_rem: + if (((word32)rax) >= ((word32)r8)) { + goto L_AES_GCMSIV_polyval_aesni_done; + } + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, rax)); + x0 = _mm_xor_si128(x0, x5); + x9 = x0; + x9 = _mm_clmulepi64_si128(x9, x1, 0); + x10 = x0; + x10 = _mm_clmulepi64_si128(x10, x1, 0x11); + x13 = x0; + x13 = _mm_clmulepi64_si128(x13, x1, 0x10); + x14 = x0; + x14 = _mm_clmulepi64_si128(x14, x1, 1); + x13 = _mm_xor_si128(x13, x14); + x14 = x13; + x14 = _mm_bslli_si128(x14, 8); + x13 = _mm_bsrli_si128(x13, 8); + x9 = _mm_xor_si128(x9, x14); + x10 = _mm_xor_si128(x10, x13); + x5 = x9; + x5 = _mm_srli_epi32(x5, 31); + x6 = x10; + x6 = _mm_srli_epi32(x6, 31); + x9 = _mm_slli_epi32(x9, 1); + x10 = _mm_slli_epi32(x10, 1); + x7 = x5; + x7 = _mm_bsrli_si128(x7, 12); + x6 = _mm_bslli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 4); + x9 = _mm_or_si128(x9, x5); + x10 = _mm_or_si128(x10, x6); + x10 = _mm_or_si128(x10, x7); + x5 = x9; + x5 = _mm_slli_epi32(x5, 31); + x6 = x9; + x6 = _mm_slli_epi32(x6, 30); + x7 = x9; + x7 = _mm_slli_epi32(x7, 25); + x5 = _mm_xor_si128(x5, x6); + x5 = _mm_xor_si128(x5, x7); + x6 = x5; + x6 = _mm_bsrli_si128(x6, 4); + x5 = _mm_bslli_si128(x5, 12); + x9 = _mm_xor_si128(x9, x5); + x8 = x9; + x8 = _mm_srli_epi32(x8, 1); + x11 = x9; + x11 = _mm_srli_epi32(x11, 2); + x12 = x9; + x12 = _mm_srli_epi32(x12, 7); + x8 = _mm_xor_si128(x8, x11); + x8 = _mm_xor_si128(x8, x12); + x8 = _mm_xor_si128(x8, x6); + x9 = _mm_xor_si128(x9, x8); + x10 = _mm_xor_si128(x10, x9); + x0 = x10; + rax = (word32)((word32)rax + 0x10); + goto L_AES_GCMSIV_polyval_aesni_rem; +L_AES_GCMSIV_polyval_aesni_done: + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_siv_bswap_mask, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); +} + +XALIGNED(16) static const word64 L_aes_gcmsiv_ctr_aesni_one[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_GCMSIV_ctr_aesni(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11 = 0, r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7, x8; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + r9 = (word64)(size_t)ctr; + + x8 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcmsiv_ctr_aesni_one, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x40; + r10 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_GCMSIV_ctr_aesni_done_64; + } + r10 = (word32)((word32)r10 & 0xffffffc0); +L_AES_GCMSIV_ctr_aesni_enc_64: + /* 64 bytes of input */ + /* siv_ctr_enc_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + x0 = x7; + x7 = _mm_add_epi32(x7, x8); + x1 = x7; + x7 = _mm_add_epi32(x7, x8); + x2 = x7; + x7 = _mm_add_epi32(x7, x8); + x3 = x7; + x7 = _mm_add_epi32(x7, x8); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_xor_si128(x0, x4); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x4); + x3 = _mm_xor_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf3 = (word32)r8; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCMSIV_ctr_aesni_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf5 = (word32)r8; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCMSIV_ctr_aesni_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_GCMSIV_ctr_aesni_64_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x4); + x1 = _mm_aesenclast_si128(x1, x4); + x2 = _mm_aesenclast_si128(x2, x4); + x3 = _mm_aesenclast_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x0 = _mm_xor_si128(x0, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 16)); + x1 = _mm_xor_si128(x1, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 32)); + x2 = _mm_xor_si128(x2, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 48)); + x3 = _mm_xor_si128(x3, x4); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r12, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r12, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r12, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_aesni_enc_64; + } +L_AES_GCMSIV_ctr_aesni_done_64: + zf7 = (word32)rax; + zf8 = (word32)rdx; + r10 = (word32)((word32)rdx); + if ((zf7) == (zf8)) { + goto L_AES_GCMSIV_ctr_aesni_done_enc; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_GCMSIV_ctr_aesni_enc_16: + /* 16 bytes of input */ + x0 = x7; + x7 = _mm_add_epi32(x7, x8); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf9 = (word32)r8; + zf10 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCMSIV_ctr_aesni_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf11 = (word32)r8; + zf12 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCMSIV_ctr_aesni_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_GCMSIV_ctr_aesni_16_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r11 = (word64)(rdi + rax); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x0 = _mm_xor_si128(x0, x4); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_aesni_enc_16; + } +L_AES_GCMSIV_ctr_aesni_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x7); +} + +#endif /* WOLFSSL_AESGCM_SIV */ +#ifdef HAVE_INTEL_AVX1 +XALIGNED(16) static const word64 L_GCM_generate_m0_avx1_rev8[] + WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_GCM_generate_m0_avx1_mod2_128[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0xe100000000000000ULL, +}; + +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL void GCM_generate_m0_avx1(const unsigned char* h, + unsigned char* m) +{ + word64 rdi, rsi; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10; + + rdi = (word64)(size_t)h; + rsi = (word64)(size_t)m; + + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_GCM_generate_m0_avx1_rev8, 0)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(L_GCM_generate_m0_avx1_mod2_128, + 0)); + x8 = _mm_setzero_si128(); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x8); + x8 = x0; + x0 = _mm_shuffle_epi8(x0, x9); + x5 = _mm_slli_epi64(x0, 63); + x4 = _mm_srli_epi64(x0, 1); + x1 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x1 = _mm_shuffle_epi32(x1, 0xff); + x4 = _mm_or_si128(x4, x5); + x1 = _mm_srai_epi32(x1, 31); + x1 = _mm_and_si128(x1, x10); + x1 = _mm_xor_si128(x1, x4); + x5 = _mm_slli_epi64(x1, 63); + x4 = _mm_srli_epi64(x1, 1); + x2 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x2 = _mm_shuffle_epi32(x2, 0xff); + x4 = _mm_or_si128(x4, x5); + x2 = _mm_srai_epi32(x2, 31); + x2 = _mm_and_si128(x2, x10); + x2 = _mm_xor_si128(x2, x4); + x5 = _mm_slli_epi64(x2, 63); + x4 = _mm_srli_epi64(x2, 1); + x3 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x3 = _mm_shuffle_epi32(x3, 0xff); + x4 = _mm_or_si128(x4, x5); + x3 = _mm_srai_epi32(x3, 31); + x3 = _mm_and_si128(x3, x10); + x3 = _mm_xor_si128(x3, x4); + x3 = _mm_shuffle_epi8(x3, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x0 = _mm_shuffle_epi8(x0, x9); + x8 = _mm_xor_si128(x3, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x3); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x1); + x4 = _mm_xor_si128(x3, x1); + x5 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x8, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 128), x0); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x3, x0); + x6 = _mm_xor_si128(x2, x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 144), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 160), x6); + x6 = _mm_xor_si128(x3, x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 176), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 192), x1); + x4 = _mm_xor_si128(x3, x1); + x5 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x8, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 208), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 224), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 240), x6); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 256), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 272), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 288), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 304), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 320), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 336), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 352), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 368), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 384), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 400), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 416), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 432), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 240)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 448), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 464), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 480), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 496), x3); +} + +XALIGNED(16) static const word64 L_avx1_aes_gcm_one[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000001ULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_two[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_three[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000003ULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_four[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_five[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000005ULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_six[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000006ULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_seven[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000007ULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_eight[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000008ULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_bswap_epi64[] WC_X64I_UNUSED = { + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_bswap_mask[] WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_avx1_aes_gcm_mod2_128[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0xc200000000000000ULL, +}; + +WC_X64I_TARGET("aes,pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_avx1(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr) +{ + word64 rdi, rsi, r12, rax, r8, r9, r11, rbx, r14, r15, r10, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6, x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[24]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + word32 zf35; + word32 zf36; + word32 zf37; + word32 zf38; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r8 = (word64)(size_t)tag; + r9 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r14 = (word64)(word64)tbytes; + r15 = (word64)(size_t)key; + r10 = (word64)(word64)nr; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 160); + x4 = _mm_setzero_si128(); + x6 = _mm_setzero_si128(); + rdx = (word32)((word32)rbx); + if (((word32)rdx) != (0xc)) { + goto L_AES_GCM_encrypt_avx1_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_loadl_epi64((const __m128i*)WC_PR(rax, 0)); + x4 = _mm_insert_epi32(x4, (int)WC_L32(rax, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x1 = _mm_xor_si128(x4, x5); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + zf1 = (word32)r10; + zf2 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + zf3 = (word32)r10; + zf4 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_avx1_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x7); + x1 = _mm_aesenclast_si128(x1, x7); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x1); + goto L_AES_GCM_encrypt_avx1_iv_done; +L_AES_GCM_encrypt_avx1_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf5 = (word32)r10; + zf6 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x9); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf7 = (word32)r10; + zf8 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x9); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_avx1_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x9); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf9 = (word32)rdx; + zf10 = 0; + rcx = (word64)(0); + if ((zf9) == (zf10)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_encrypt_avx1_calc_iv_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rax, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_done; + } +L_AES_GCM_encrypt_avx1_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_encrypt_avx1_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); +L_AES_GCM_encrypt_avx1_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x8 = _mm_xor_si128(x8, x4); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf11 = (word32)r10; + zf12 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf13 = (word32)r10; + zf14 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_avx1_calc_iv_2_aesenc_avx_last: + x8 = _mm_aesenclast_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x8); +L_AES_GCM_encrypt_avx1_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_encrypt_avx1_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_encrypt_avx1_calc_aad_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_encrypt_avx1_calc_aad_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x6, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x6, 0x11); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + x1 = _mm_xor_si128(x1, x6); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x6, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x6 = _mm_xor_si128(x6, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx1_calc_aad_16_loop; + } + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx1_calc_aad_done; + } +L_AES_GCM_encrypt_avx1_calc_aad_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_encrypt_avx1_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx1_calc_aad_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x6, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x6, 0x11); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + x1 = _mm_xor_si128(x1, x6); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x6, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x6 = _mm_xor_si128(x6, x2); +L_AES_GCM_encrypt_avx1_calc_aad_done: + /* Calculate counter and H */ + x9 = _mm_srli_epi64(x5, 63); + x8 = _mm_slli_epi64(x5, 1); + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_or_si128(x8, x9); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_mod2_128, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + x5 = _mm_xor_si128(x5, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x4); + rbx = (word32)(0); + zf15 = (word32)r9; + zf16 = 0x80; + r13 = (word32)((word32)r9); + if ((zf15) < (zf16)) { + goto L_AES_GCM_encrypt_avx1_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + x2 = x6; + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + x0 = _mm_clmulepi64_si128(x5, x5, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x0 = _mm_xor_si128(x0, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = _mm_clmulepi64_si128(x0, x5, 0x11); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x1 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x1 = _mm_xor_si128(x1, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + x3 = _mm_clmulepi64_si128(x0, x0, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x3 = _mm_xor_si128(x3, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x3); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = _mm_clmulepi64_si128(x1, x0, 0x11); + x8 = _mm_clmulepi64_si128(x1, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(x1, x1, 0); + x7 = _mm_clmulepi64_si128(x1, x1, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = _mm_clmulepi64_si128(x3, x1, 0x11); + x8 = _mm_clmulepi64_si128(x3, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(x3, x3, 0); + x7 = _mm_clmulepi64_si128(x3, x3, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + /* First 128 bytes of input */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_bswap_epi64, 0)); + x8 = _mm_shuffle_epi8(x0, x1); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf17 = (word32)r10; + zf18 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_encrypt_avx1_aesenc_128_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf19 = (word32)r10; + zf20 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_encrypt_avx1_aesenc_128_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_avx1_aesenc_128_enc_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x15); + zf21 = (word32)r13; + zf22 = 0x80; + rbx = (word32)(0x80); + if ((zf21) <= (zf22)) { + goto L_AES_GCM_encrypt_avx1_end_128; + } + /* More 128 bytes of input */ +L_AES_GCM_encrypt_avx1_ghash_128: + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_bswap_epi64, 0)); + x8 = _mm_shuffle_epi8(x0, x1); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -128)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x2); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x1 = _mm_xor_si128(x1, x7); + x5 = _mm_xor_si128(x5, x0); + x3 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x2 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x1 = _mm_clmulepi64_si128(x1, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x3); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -112)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -96)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -80)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -64)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -48)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -32)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -16)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x5 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x1); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x7 = _mm_slli_epi32(x2, 31); + x4 = _mm_slli_epi32(x2, 30); + x5 = _mm_slli_epi32(x2, 25); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_xor_si128(x7, x5); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x4 = _mm_bsrli_si128(x7, 4); + x7 = _mm_bslli_si128(x7, 12); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x2 = _mm_xor_si128(x2, x7); + x5 = _mm_srli_epi32(x2, 1); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x1 = _mm_srli_epi32(x2, 2); + x0 = _mm_srli_epi32(x2, 7); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_xor_si128(x5, x1); + x5 = _mm_xor_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_xor_si128(x5, x4); + x2 = _mm_xor_si128(x2, x5); + x2 = _mm_xor_si128(x2, x3); + zf23 = (word32)r10; + zf24 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf25 = (word32)r10; + zf26 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_avx1_aesenc_128_ghash_avx_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + rbx = (word32)((word32)rbx + 0x80); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx1_ghash_128; + } +L_AES_GCM_encrypt_avx1_end_128: + x4 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_bswap_mask, 0)); + x8 = _mm_shuffle_epi8(x8, x4); + x9 = _mm_shuffle_epi8(x9, x4); + x10 = _mm_shuffle_epi8(x10, x4); + x11 = _mm_shuffle_epi8(x11, x4); + x8 = _mm_xor_si128(x8, x2); + x12 = _mm_shuffle_epi8(x12, x4); + x13 = _mm_shuffle_epi8(x13, x4); + x14 = _mm_shuffle_epi8(x14, x4); + x15 = _mm_shuffle_epi8(x15, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x15, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = _mm_clmulepi64_si128(x7, x15, 0x11); + x0 = _mm_clmulepi64_si128(x7, x15, 0); + x1 = _mm_xor_si128(x1, x15); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x14, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x14, 0x11); + x0 = _mm_clmulepi64_si128(x5, x14, 0); + x1 = _mm_xor_si128(x1, x14); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x13, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = _mm_clmulepi64_si128(x7, x13, 0x11); + x0 = _mm_clmulepi64_si128(x7, x13, 0); + x1 = _mm_xor_si128(x1, x13); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x12, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x12, 0x11); + x0 = _mm_clmulepi64_si128(x5, x12, 0); + x1 = _mm_xor_si128(x1, x12); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x11, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = _mm_clmulepi64_si128(x7, x11, 0x11); + x0 = _mm_clmulepi64_si128(x7, x11, 0); + x1 = _mm_xor_si128(x1, x11); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x10, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x10, 0x11); + x0 = _mm_clmulepi64_si128(x5, x10, 0); + x1 = _mm_xor_si128(x1, x10); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x9, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = _mm_clmulepi64_si128(x7, x9, 0x11); + x0 = _mm_clmulepi64_si128(x7, x9, 0); + x1 = _mm_xor_si128(x1, x9); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x8, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x8, 0x11); + x0 = _mm_clmulepi64_si128(x5, x8, 0); + x1 = _mm_xor_si128(x1, x8); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = _mm_slli_epi32(x4, 31); + x1 = _mm_slli_epi32(x4, 30); + x2 = _mm_slli_epi32(x4, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x4 = _mm_xor_si128(x4, x0); + x2 = _mm_srli_epi32(x4, 1); + x3 = _mm_srli_epi32(x4, 2); + x0 = _mm_srli_epi32(x4, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x4); + x6 = _mm_xor_si128(x6, x2); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_encrypt_avx1_done_128: + rdx = (word32)((word32)r9); + if (((word32)rbx) >= ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx1_done_enc; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx1_last_block_done; + } + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x8 = _mm_shuffle_epi8(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x9); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf27 = (word32)r10; + zf28 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_encrypt_avx1_aesenc_block_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf29 = (word32)r10; + zf30 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_encrypt_avx1_aesenc_block_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_avx1_aesenc_block_last: + x8 = _mm_aesenclast_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rbx)); + x8 = _mm_xor_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, rbx), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx1_last_block_ghash; + } +L_AES_GCM_encrypt_avx1_last_block_start: + x13 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rbx)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x8 = _mm_shuffle_epi8(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x9); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x10 = _mm_clmulepi64_si128(x6, x5, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x11 = _mm_clmulepi64_si128(x6, x5, 1); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x12 = _mm_clmulepi64_si128(x6, x5, 0); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x1 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x10 = _mm_xor_si128(x10, x11); + x2 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x2 = _mm_xor_si128(x2, x12); + x3 = _mm_xor_si128(x1, x10); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_mod2_128, 0)); + x11 = _mm_clmulepi64_si128(x2, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x10 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_clmulepi64_si128(x10, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x6 = _mm_xor_si128(x10, x3); + zf31 = (word32)r10; + zf32 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_GCM_encrypt_avx1_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf33 = (word32)r10; + zf34 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_GCM_encrypt_avx1_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_avx1_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, x9); + x0 = x13; + x8 = _mm_xor_si128(x8, x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, rbx), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + rbx = (word32)((word32)rbx + 0x10); + x6 = _mm_xor_si128(x6, x8); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx1_last_block_start; + } +L_AES_GCM_encrypt_avx1_last_block_ghash: + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); +L_AES_GCM_encrypt_avx1_last_block_done: + rcx = (word32)((word32)r9); + rdx = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_done; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x4 = _mm_xor_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf35 = (word32)r10; + zf36 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf35) < (sword32)(zf36)) { + goto L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x9); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf37 = (word32)r10; + zf38 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf37) < (sword32)(zf38)) { + goto L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x9); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_aesenc_avx_last: + x4 = _mm_aesenclast_si128(x4, x9); + rsp = (word64)(rsp - 16); + rcx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(rsi, rbx) = (byte)((byte)r13); + WC_S8(rsp, rcx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_loop; + } + r13 = (word64)(0); + if (((word32)rcx) == (0x10)) { + goto L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_finish_enc; + } +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_byte_loop: + WC_S8(rsp, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) < (0x10)) { + goto L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_byte_loop; + } +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_finish_enc: + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x4); + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); +L_AES_GCM_encrypt_avx1_aesenc_last15_enc_avx_done: +L_AES_GCM_encrypt_avx1_done_enc: + rdx = (word32)((word32)r9); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x1 = _mm_cvtsi64_si128((long long)rcx); + x0 = _mm_unpacklo_epi64(x0, x1); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + x0 = _mm_xor_si128(x0, x6); + if (((word32)r14) == (0x10)) { + goto L_AES_GCM_encrypt_avx1_store_tag_16; + } + rcx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_avx1_store_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + WC_S8(r8, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)r14)) { + goto L_AES_GCM_encrypt_avx1_store_tag_loop; + } + goto L_AES_GCM_encrypt_avx1_store_tag_done; +L_AES_GCM_encrypt_avx1_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x0); +L_AES_GCM_encrypt_avx1_store_tag_done: + ; +} + +WC_X64I_TARGET("aes,pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_decrypt_avx1(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res) +{ + word64 rdi, rsi, r12, rax, r8, r9, r11, rbx, r14, r15, r10, rbp, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6, x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[28]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r8 = (word64)(size_t)tag; + r9 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r14 = (word64)(word64)tbytes; + r15 = (word64)(size_t)key; + r10 = (word64)(word64)nr; + rbp = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 224; + rsp = (word64)(rsp - 168); + x4 = _mm_setzero_si128(); + x6 = _mm_setzero_si128(); + zf1 = (word32)rbx; + zf2 = 0xc; + rdx = (word32)((word32)rbx); + if ((zf1) != (zf2)) { + goto L_AES_GCM_decrypt_avx1_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_loadl_epi64((const __m128i*)WC_PR(rax, 0)); + x4 = _mm_insert_epi32(x4, (int)WC_L32(rax, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x1 = _mm_xor_si128(x4, x5); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + zf3 = (word32)r10; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + zf5 = (word32)r10; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x5 = _mm_aesenc_si128(x5, x7); + x1 = _mm_aesenc_si128(x1, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_avx1_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x7); + x1 = _mm_aesenclast_si128(x1, x7); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x1); + goto L_AES_GCM_decrypt_avx1_iv_done; +L_AES_GCM_decrypt_avx1_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf7 = (word32)r10; + zf8 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x9); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf9 = (word32)r10; + zf10 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x9); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_avx1_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x9); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf11 = (word32)rdx; + zf12 = 0; + rcx = (word64)(0); + if ((zf11) == (zf12)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_decrypt_avx1_calc_iv_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rax, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_done; + } +L_AES_GCM_decrypt_avx1_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_decrypt_avx1_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); +L_AES_GCM_decrypt_avx1_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x4 = _mm_xor_si128(x4, x2); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x8 = _mm_xor_si128(x8, x4); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf13 = (word32)r10; + zf14 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf15 = (word32)r10; + zf16 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_avx1_calc_iv_2_aesenc_avx_last: + x8 = _mm_aesenclast_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x8); +L_AES_GCM_decrypt_avx1_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_decrypt_avx1_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_decrypt_avx1_calc_aad_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_decrypt_avx1_calc_aad_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x6, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x6, 0x11); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + x1 = _mm_xor_si128(x1, x6); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x6, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x6 = _mm_xor_si128(x6, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx1_calc_aad_16_loop; + } + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx1_calc_aad_done; + } +L_AES_GCM_decrypt_avx1_calc_aad_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_decrypt_avx1_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx1_calc_aad_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x6, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x6, 0x11); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + x1 = _mm_xor_si128(x1, x6); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x7 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x7 = _mm_xor_si128(x7, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x6, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + x0 = _mm_slli_epi32(x7, 31); + x1 = _mm_slli_epi32(x7, 30); + x2 = _mm_slli_epi32(x7, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + x2 = _mm_srli_epi32(x7, 1); + x3 = _mm_srli_epi32(x7, 2); + x0 = _mm_srli_epi32(x7, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x7); + x6 = _mm_xor_si128(x6, x2); +L_AES_GCM_decrypt_avx1_calc_aad_done: + /* Calculate counter and H */ + x9 = _mm_srli_epi64(x5, 63); + x8 = _mm_slli_epi64(x5, 1); + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_or_si128(x8, x9); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_mod2_128, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + x5 = _mm_xor_si128(x5, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x4); + rbx = (word32)(0); + zf17 = (word32)r9; + zf18 = 0x80; + r13 = (word32)((word32)r9); + if ((zf17) < (zf18)) { + goto L_AES_GCM_decrypt_avx1_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + x2 = x6; + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + x0 = _mm_clmulepi64_si128(x5, x5, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x0 = _mm_xor_si128(x0, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = _mm_clmulepi64_si128(x0, x5, 0x11); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x1 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x1 = _mm_xor_si128(x1, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + x3 = _mm_clmulepi64_si128(x0, x0, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x3 = _mm_xor_si128(x3, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x3); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = _mm_clmulepi64_si128(x1, x0, 0x11); + x8 = _mm_clmulepi64_si128(x1, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(x1, x1, 0); + x7 = _mm_clmulepi64_si128(x1, x1, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = _mm_clmulepi64_si128(x3, x1, 0x11); + x8 = _mm_clmulepi64_si128(x3, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(x3, x3, 0); + x7 = _mm_clmulepi64_si128(x3, x3, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); +L_AES_GCM_decrypt_avx1_ghash_128: + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_bswap_epi64, 0)); + x8 = _mm_shuffle_epi8(x0, x1); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x2); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x1 = _mm_xor_si128(x1, x7); + x5 = _mm_xor_si128(x5, x0); + x3 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x2 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x1 = _mm_clmulepi64_si128(x1, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x3); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x5 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x1); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x7 = _mm_slli_epi32(x2, 31); + x4 = _mm_slli_epi32(x2, 30); + x5 = _mm_slli_epi32(x2, 25); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_xor_si128(x7, x5); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x4 = _mm_bsrli_si128(x7, 4); + x7 = _mm_bslli_si128(x7, 12); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x2 = _mm_xor_si128(x2, x7); + x5 = _mm_srli_epi32(x2, 1); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x1 = _mm_srli_epi32(x2, 2); + x0 = _mm_srli_epi32(x2, 7); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_xor_si128(x5, x1); + x5 = _mm_xor_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + x5 = _mm_xor_si128(x5, x4); + x2 = _mm_xor_si128(x2, x5); + x2 = _mm_xor_si128(x2, x3); + zf19 = (word32)r10; + zf20 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf21 = (word32)r10; + zf22 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_avx1_aesenc_128_ghash_avx_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + rbx = (word32)((word32)rbx + 0x80); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_avx1_ghash_128; + } + x6 = x2; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_decrypt_avx1_done_128: + rdx = (word32)((word32)r9); + if (((word32)rbx) >= ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx1_done_dec; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_avx1_last_block_done; + } +L_AES_GCM_decrypt_avx1_last_block_start: + x13 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rbx)); + x0 = x5; + x1 = _mm_shuffle_epi8(x13, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x1 = _mm_xor_si128(x1, x6); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x8 = _mm_shuffle_epi8(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x9); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x10 = _mm_clmulepi64_si128(x1, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x11 = _mm_clmulepi64_si128(x1, x0, 1); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x12 = _mm_clmulepi64_si128(x1, x0, 0); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x10 = _mm_xor_si128(x10, x11); + x2 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x2 = _mm_xor_si128(x2, x12); + x3 = _mm_xor_si128(x1, x10); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_mod2_128, 0)); + x11 = _mm_clmulepi64_si128(x2, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x10 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_clmulepi64_si128(x10, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x6 = _mm_xor_si128(x10, x3); + zf23 = (word32)r10; + zf24 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_decrypt_avx1_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf25 = (word32)r10; + zf26 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_decrypt_avx1_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_avx1_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, x9); + x0 = x13; + x8 = _mm_xor_si128(x8, x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, rbx), x8); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_avx1_last_block_start; + } +L_AES_GCM_decrypt_avx1_last_block_done: + rcx = (word32)((word32)r9); + rdx = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_done; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x4 = _mm_xor_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf27 = (word32)r10; + zf28 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x9); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf29 = (word32)r10; + zf30 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x9); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_aesenc_avx_last: + x4 = _mm_aesenclast_si128(x4, x9); + rsp = (word64)(rsp - 32); + rcx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + x0 = _mm_setzero_si128(); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); +L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + WC_S8(rsp, rcx + 16) = (byte)((byte)r13); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(rsi, rbx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_loop; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + rsp = (word64)(rsp + 32); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x4); + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); +L_AES_GCM_decrypt_avx1_aesenc_last15_dec_avx_done: +L_AES_GCM_decrypt_avx1_done_dec: + rdx = (word32)((word32)r9); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x1 = _mm_cvtsi64_si128((long long)rcx); + x0 = _mm_unpacklo_epi64(x0, x1); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + x0 = _mm_xor_si128(x0, x6); + if (((word32)r14) == (0x10)) { + goto L_AES_GCM_decrypt_avx1_cmp_tag_16; + } + rsp = (word64)(rsp - 16); + rcx = (word64)(0); + rbx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_avx1_cmp_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(r8, + rcx)) & 0xff); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)((byte)rbx | ( + byte)r13) & 0xff); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)r14)) { + goto L_AES_GCM_decrypt_avx1_cmp_tag_loop; + } + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)(((byte)rbx) == (0)) & 0xff); + rsp = (word64)(rsp + 16); + rcx = (word64)(0); + goto L_AES_GCM_decrypt_avx1_cmp_tag_done; +L_AES_GCM_decrypt_avx1_cmp_tag_16: + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x0 = _mm_cmpeq_epi8(x0, x1); + rdx = (word32)((word32)_mm_movemask_epi8(x0)); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + rbx = (word32)(0); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)(((word32)rdx) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_avx1_cmp_tag_done: + WC_S32(rbp, 0) = (word32)((word32)rbx); +} + +#ifdef WOLFSSL_AESGCM_STREAM +WC_X64I_TARGET("aes,pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_init_avx1(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, rsp, rdx, rcx = 0, r13 = 0, + r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)ivec; + r11 = (word64)(word32)ibytes; + r8 = (word64)(size_t)h; + r9 = (word64)(size_t)counter; + rax = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_setzero_si128(); + rdx = (word32)((word32)r11); + if (((word32)rdx) != (0xc)) { + goto L_AES_GCM_init_avx1_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_loadl_epi64((const __m128i*)WC_PR(r10, 0)); + x4 = _mm_insert_epi32(x4, (int)WC_L32(r10, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_xor_si128(x4, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + zf1 = (word32)rsi; + zf2 = 0xb; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_init_avx1_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + zf3 = (word32)rsi; + zf4 = 0xd; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_init_avx1_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_avx1_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x6); + x1 = _mm_aesenclast_si128(x1, x6); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x15 = x1; + goto L_AES_GCM_init_avx1_iv_done; +L_AES_GCM_init_avx1_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf5 = (word32)rsi; + zf6 = 0xb; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_init_avx1_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x8); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf7 = (word32)rsi; + zf8 = 0xd; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_init_avx1_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x8); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_avx1_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x8); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf9 = (word32)rdx; + zf10 = 0; + rcx = (word64)(0); + if ((zf9) == (zf10)) { + goto L_AES_GCM_init_avx1_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_init_avx1_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_init_avx1_calc_iv_16_loop: + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, rcx)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_avx1_calc_iv_16_loop; + } + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_init_avx1_calc_iv_done; + } +L_AES_GCM_init_avx1_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x7 = _mm_setzero_si128(); + r13 = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x7); +L_AES_GCM_init_avx1_calc_iv_loop: + r12 = (word32)((word32)WC_L8(r10, rcx)); + WC_S8(rsp, r13) = (byte)((byte)r12); + rcx = (word32)((word32)rcx + 1); + r13 = (word32)((word32)r13 + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_avx1_calc_iv_loop; + } + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); +L_AES_GCM_init_avx1_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf11 = (word32)rsi; + zf12 = 0xb; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_init_avx1_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x8); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf13 = (word32)rsi; + zf14 = 0xd; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_init_avx1_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x8); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_avx1_calc_iv_2_aesenc_avx_last: + x7 = _mm_aesenclast_si128(x7, x8); + x15 = x7; +L_AES_GCM_init_avx1_iv_done: + _mm_storeu_si128((__m128i*)WC_PW(rax, 0), x15); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x4); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_aad_update_avx1(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5, x6, x7 = _mm_setzero_si128(); + + rdi = (word64)(size_t)addt; + rsi = (word64)(word32)abytes; + rdx = (word64)(size_t)tag; + rax = (word64)(size_t)h; + + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + rcx = (word32)(0); +L_AES_GCM_aad_update_avx1_16_loop: + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rcx)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x5, 0x4e); + x2 = _mm_shuffle_epi32(x6, 0x4e); + x3 = _mm_clmulepi64_si128(x6, x5, 0x11); + x0 = _mm_clmulepi64_si128(x6, x5, 0); + x1 = _mm_xor_si128(x1, x5); + x2 = _mm_xor_si128(x2, x6); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = x0; + x5 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x5 = _mm_xor_si128(x5, x1); + x0 = _mm_srli_epi32(x4, 31); + x1 = _mm_srli_epi32(x5, 31); + x4 = _mm_slli_epi32(x4, 1); + x5 = _mm_slli_epi32(x5, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x5 = _mm_or_si128(x5, x2); + x4 = _mm_or_si128(x4, x0); + x5 = _mm_or_si128(x5, x1); + x0 = _mm_slli_epi32(x4, 31); + x1 = _mm_slli_epi32(x4, 30); + x2 = _mm_slli_epi32(x4, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x4 = _mm_xor_si128(x4, x0); + x2 = _mm_srli_epi32(x4, 1); + x3 = _mm_srli_epi32(x4, 2); + x0 = _mm_srli_epi32(x4, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x4); + x5 = _mm_xor_si128(x5, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rsi)) { + goto L_AES_GCM_aad_update_avx1_16_loop; + } + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x5); +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_block_avx1(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8; + __m128i x0, x1; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(size_t)counter; + + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x0 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x1); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf1 = (word32)rsi; + zf2 = 0xb; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_block_avx1_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf3 = (word32)rsi; + zf4 = 0xd; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_block_avx1_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_block_avx1_aesenc_block_last: + x0 = _mm_aesenclast_si128(x0, x1); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_ghash_block_avx1(const unsigned char* addt, + unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, rdx; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + + rdi = (word64)(size_t)addt; + rsi = (word64)(size_t)tag; + rdx = (word64)(size_t)h; + + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x4); +} + +WC_X64I_TARGET("aes,pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_update_avx1(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, r13, rcx = 0, + rdx = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5, x6, x7 = _mm_setzero_si128(), x8, + x9, x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[20]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 160); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x9 = _mm_srli_epi64(x5, 63); + x8 = _mm_slli_epi64(x5, 1); + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_or_si128(x8, x9); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + r14 = (word32)(0); + zf1 = (word32)r8; + zf2 = 0x80; + r13 = (word32)((word32)r8); + if ((zf1) < (zf2)) { + goto L_AES_GCM_encrypt_update_avx1_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + x2 = x6; + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + x0 = _mm_clmulepi64_si128(x5, x5, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x0 = _mm_xor_si128(x0, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = _mm_clmulepi64_si128(x0, x5, 0x11); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x1 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x1 = _mm_xor_si128(x1, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + x3 = _mm_clmulepi64_si128(x0, x0, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x3 = _mm_xor_si128(x3, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x3); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = _mm_clmulepi64_si128(x1, x0, 0x11); + x8 = _mm_clmulepi64_si128(x1, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(x1, x1, 0); + x7 = _mm_clmulepi64_si128(x1, x1, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = _mm_clmulepi64_si128(x3, x1, 0x11); + x8 = _mm_clmulepi64_si128(x3, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(x3, x3, 0); + x7 = _mm_clmulepi64_si128(x3, x3, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + /* First 128 bytes of input */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_bswap_epi64, 0)); + x8 = _mm_shuffle_epi8(x0, x1); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf3 = (word32)rsi; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_update_avx1_aesenc_128_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf5 = (word32)rsi; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_encrypt_update_avx1_aesenc_128_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_avx1_aesenc_128_enc_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(r10, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(r10, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(r10, 112), x15); + zf7 = (word32)r13; + zf8 = 0x80; + r14 = (word32)(0x80); + if ((zf7) <= (zf8)) { + goto L_AES_GCM_encrypt_update_avx1_end_128; + } + /* More 128 bytes of input */ +L_AES_GCM_encrypt_update_avx1_ghash_128: + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_bswap_epi64, 0)); + x8 = _mm_shuffle_epi8(x0, x1); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -128)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x2); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x1 = _mm_xor_si128(x1, x7); + x5 = _mm_xor_si128(x5, x0); + x3 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x2 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x1 = _mm_clmulepi64_si128(x1, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x3); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -112)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -96)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -80)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -64)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -48)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -32)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -16)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x5 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x1); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x7 = _mm_slli_epi32(x2, 31); + x4 = _mm_slli_epi32(x2, 30); + x5 = _mm_slli_epi32(x2, 25); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_xor_si128(x7, x5); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x4 = _mm_bsrli_si128(x7, 4); + x7 = _mm_bslli_si128(x7, 12); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x2 = _mm_xor_si128(x2, x7); + x5 = _mm_srli_epi32(x2, 1); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x1 = _mm_srli_epi32(x2, 2); + x0 = _mm_srli_epi32(x2, 7); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_xor_si128(x5, x1); + x5 = _mm_xor_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_xor_si128(x5, x4); + x2 = _mm_xor_si128(x2, x5); + x2 = _mm_xor_si128(x2, x3); + zf9 = (word32)rsi; + zf10 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_encrypt_update_avx1_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf11 = (word32)rsi; + zf12 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_encrypt_update_avx1_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_avx1_aesenc_128_ghash_avx_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + r14 = (word32)((word32)r14 + 0x80); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx1_ghash_128; + } +L_AES_GCM_encrypt_update_avx1_end_128: + x4 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_bswap_mask, 0)); + x8 = _mm_shuffle_epi8(x8, x4); + x9 = _mm_shuffle_epi8(x9, x4); + x10 = _mm_shuffle_epi8(x10, x4); + x11 = _mm_shuffle_epi8(x11, x4); + x8 = _mm_xor_si128(x8, x2); + x12 = _mm_shuffle_epi8(x12, x4); + x13 = _mm_shuffle_epi8(x13, x4); + x14 = _mm_shuffle_epi8(x14, x4); + x15 = _mm_shuffle_epi8(x15, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x15, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = _mm_clmulepi64_si128(x7, x15, 0x11); + x0 = _mm_clmulepi64_si128(x7, x15, 0); + x1 = _mm_xor_si128(x1, x15); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = x0; + x6 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x14, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x14, 0x11); + x0 = _mm_clmulepi64_si128(x5, x14, 0); + x1 = _mm_xor_si128(x1, x14); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x13, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = _mm_clmulepi64_si128(x7, x13, 0x11); + x0 = _mm_clmulepi64_si128(x7, x13, 0); + x1 = _mm_xor_si128(x1, x13); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x12, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x12, 0x11); + x0 = _mm_clmulepi64_si128(x5, x12, 0); + x1 = _mm_xor_si128(x1, x12); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x11, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = _mm_clmulepi64_si128(x7, x11, 0x11); + x0 = _mm_clmulepi64_si128(x7, x11, 0); + x1 = _mm_xor_si128(x1, x11); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x10, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x10, 0x11); + x0 = _mm_clmulepi64_si128(x5, x10, 0); + x1 = _mm_xor_si128(x1, x10); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x9, 0x4e); + x2 = _mm_shuffle_epi32(x7, 0x4e); + x3 = _mm_clmulepi64_si128(x7, x9, 0x11); + x0 = _mm_clmulepi64_si128(x7, x9, 0); + x1 = _mm_xor_si128(x1, x9); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + /* ghash_gfmul_xor_avx */ + x1 = _mm_shuffle_epi32(x8, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x8, 0x11); + x0 = _mm_clmulepi64_si128(x5, x8, 0); + x1 = _mm_xor_si128(x1, x8); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x4 = _mm_xor_si128(x4, x0); + x6 = _mm_xor_si128(x6, x3); + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x4 = _mm_xor_si128(x4, x2); + x6 = _mm_xor_si128(x6, x1); + x0 = _mm_slli_epi32(x4, 31); + x1 = _mm_slli_epi32(x4, 30); + x2 = _mm_slli_epi32(x4, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x4 = _mm_xor_si128(x4, x0); + x2 = _mm_srli_epi32(x4, 1); + x3 = _mm_srli_epi32(x4, 2); + x0 = _mm_srli_epi32(x4, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x4); + x6 = _mm_xor_si128(x6, x2); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_encrypt_update_avx1_done_128: + rdx = (word32)((word32)r8); + if (((word32)r14) >= ((word32)rdx)) { + goto L_AES_GCM_encrypt_update_avx1_done_enc; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx1_last_block_done; + } + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_shuffle_epi8(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x9); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf13 = (word32)rsi; + zf14 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_update_avx1_aesenc_block_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf15 = (word32)rsi; + zf16 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_encrypt_update_avx1_aesenc_block_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_avx1_aesenc_block_last: + x8 = _mm_aesenclast_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r11, r14)); + x8 = _mm_xor_si128(x8, x9); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx1_last_block_ghash; + } +L_AES_GCM_encrypt_update_avx1_last_block_start: + x13 = _mm_loadu_si128((const __m128i*)WC_PR(r11, r14)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_shuffle_epi8(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x9); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x10 = _mm_clmulepi64_si128(x6, x5, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x11 = _mm_clmulepi64_si128(x6, x5, 1); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x12 = _mm_clmulepi64_si128(x6, x5, 0); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x1 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x10 = _mm_xor_si128(x10, x11); + x2 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x2 = _mm_xor_si128(x2, x12); + x3 = _mm_xor_si128(x1, x10); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_mod2_128, 0)); + x11 = _mm_clmulepi64_si128(x2, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x10 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_clmulepi64_si128(x10, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x6 = _mm_xor_si128(x10, x3); + zf17 = (word32)rsi; + zf18 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_encrypt_update_avx1_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf19 = (word32)rsi; + zf20 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_encrypt_update_avx1_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_avx1_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, x9); + x0 = x13; + x8 = _mm_xor_si128(x8, x0); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + r14 = (word32)((word32)r14 + 0x10); + x6 = _mm_xor_si128(x6, x8); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx1_last_block_start; + } +L_AES_GCM_encrypt_update_avx1_last_block_ghash: + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x6, 0x4e); + x11 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x6); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x6 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x6 = _mm_xor_si128(x6, x14); +L_AES_GCM_encrypt_update_avx1_last_block_done: +L_AES_GCM_encrypt_update_avx1_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x6); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_final_avx1(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr) +{ + word64 rdi, rsi, rax, r10, r11, r9, r8, rsp, rdx, rcx, r13 = 0; + __m128i x0, x1, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rax = (word64)(word32)tbytes; + r10 = (word64)(word32)nbytes; + r11 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + r8 = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_srli_epi64(x5, 63); + x7 = _mm_slli_epi64(x5, 1); + x8 = _mm_bslli_si128(x8, 8); + x7 = _mm_or_si128(x7, x8); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x7); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x1 = _mm_cvtsi64_si128((long long)rcx); + x0 = _mm_unpacklo_epi64(x0, x1); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_red_avx */ + x8 = _mm_shuffle_epi32(x5, 0x4e); + x9 = _mm_shuffle_epi32(x4, 0x4e); + x10 = _mm_clmulepi64_si128(x4, x5, 0x11); + x7 = _mm_clmulepi64_si128(x4, x5, 0); + x8 = _mm_xor_si128(x8, x5); + x9 = _mm_xor_si128(x9, x4); + x8 = _mm_clmulepi64_si128(x8, x9, 0); + x8 = _mm_xor_si128(x8, x7); + x8 = _mm_xor_si128(x8, x10); + x9 = _mm_bslli_si128(x8, 8); + x8 = _mm_bsrli_si128(x8, 8); + x7 = _mm_xor_si128(x7, x9); + x4 = _mm_xor_si128(x10, x8); + x11 = _mm_slli_epi32(x7, 31); + x12 = _mm_slli_epi32(x7, 30); + x13 = _mm_slli_epi32(x7, 25); + x11 = _mm_xor_si128(x11, x12); + x11 = _mm_xor_si128(x11, x13); + x12 = _mm_bsrli_si128(x11, 4); + x11 = _mm_bslli_si128(x11, 12); + x7 = _mm_xor_si128(x7, x11); + x13 = _mm_srli_epi32(x7, 1); + x9 = _mm_srli_epi32(x7, 2); + x8 = _mm_srli_epi32(x7, 7); + x13 = _mm_xor_si128(x13, x9); + x13 = _mm_xor_si128(x13, x8); + x13 = _mm_xor_si128(x13, x12); + x13 = _mm_xor_si128(x13, x7); + x4 = _mm_xor_si128(x4, x13); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x4, x6); + if (((word32)rax) == (0x10)) { + goto L_AES_GCM_encrypt_final_avx1_store_tag_16; + } + rcx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_final_avx1_store_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + WC_S8(rsi, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)rax)) { + goto L_AES_GCM_encrypt_final_avx1_store_tag_loop; + } + goto L_AES_GCM_encrypt_final_avx1_store_tag_done; +L_AES_GCM_encrypt_final_avx1_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); +L_AES_GCM_encrypt_final_avx1_store_tag_done: + ; +} + +WC_X64I_TARGET("aes,pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_decrypt_update_avx1(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, r13, rcx = 0, + rdx = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5, x6, x7 = _mm_setzero_si128(), x8, + x9, x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[24]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 168); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x9 = _mm_srli_epi64(x5, 63); + x8 = _mm_slli_epi64(x5, 1); + x9 = _mm_bslli_si128(x9, 8); + x8 = _mm_or_si128(x8, x9); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + r14 = (word32)(0); + zf1 = (word32)r8; + zf2 = 0x80; + r13 = (word32)((word32)r8); + if ((zf1) < (zf2)) { + goto L_AES_GCM_decrypt_update_avx1_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + x2 = x6; + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + x0 = _mm_clmulepi64_si128(x5, x5, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x0 = _mm_xor_si128(x0, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x5, 0x4e); + x10 = _mm_shuffle_epi32(x0, 0x4e); + x11 = _mm_clmulepi64_si128(x0, x5, 0x11); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + x9 = _mm_xor_si128(x9, x5); + x10 = _mm_xor_si128(x10, x0); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x1 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x1 = _mm_xor_si128(x1, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + x3 = _mm_clmulepi64_si128(x0, x0, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x3 = _mm_xor_si128(x3, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x3); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_shuffle_epi32(x1, 0x4e); + x11 = _mm_clmulepi64_si128(x1, x0, 0x11); + x8 = _mm_clmulepi64_si128(x1, x0, 0); + x9 = _mm_xor_si128(x9, x0); + x10 = _mm_xor_si128(x10, x1); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(x1, x1, 0); + x7 = _mm_clmulepi64_si128(x1, x1, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + x9 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_shuffle_epi32(x3, 0x4e); + x11 = _mm_clmulepi64_si128(x3, x1, 0x11); + x8 = _mm_clmulepi64_si128(x3, x1, 0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x3); + x9 = _mm_clmulepi64_si128(x9, x10, 0); + x9 = _mm_xor_si128(x9, x8); + x9 = _mm_xor_si128(x9, x11); + x10 = _mm_bslli_si128(x9, 8); + x9 = _mm_bsrli_si128(x9, 8); + x8 = _mm_xor_si128(x8, x10); + x7 = _mm_xor_si128(x11, x9); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(x3, x3, 0); + x7 = _mm_clmulepi64_si128(x3, x3, 0x11); + x12 = _mm_slli_epi32(x8, 31); + x13 = _mm_slli_epi32(x8, 30); + x14 = _mm_slli_epi32(x8, 25); + x12 = _mm_xor_si128(x12, x13); + x12 = _mm_xor_si128(x12, x14); + x13 = _mm_bsrli_si128(x12, 4); + x12 = _mm_bslli_si128(x12, 12); + x8 = _mm_xor_si128(x8, x12); + x14 = _mm_srli_epi32(x8, 1); + x10 = _mm_srli_epi32(x8, 2); + x9 = _mm_srli_epi32(x8, 7); + x14 = _mm_xor_si128(x14, x10); + x14 = _mm_xor_si128(x14, x9); + x14 = _mm_xor_si128(x14, x13); + x14 = _mm_xor_si128(x14, x8); + x7 = _mm_xor_si128(x7, x14); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); +L_AES_GCM_decrypt_update_avx1_ghash_128: + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_bswap_epi64, 0)); + x8 = _mm_shuffle_epi8(x0, x1); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_two, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_three, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_four, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_five, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_six, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_seven, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_eight, 0))); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x2); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x1 = _mm_xor_si128(x1, x7); + x5 = _mm_xor_si128(x5, x0); + x3 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x2 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x1 = _mm_clmulepi64_si128(x1, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x3); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x4 = _mm_shuffle_epi32(x7, 0x4e); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x4 = _mm_xor_si128(x4, x7); + x5 = _mm_shuffle_epi32(x0, 0x4e); + x5 = _mm_xor_si128(x5, x0); + x6 = _mm_clmulepi64_si128(x0, x7, 0x11); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x7 = _mm_clmulepi64_si128(x0, x7, 0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x4 = _mm_clmulepi64_si128(x4, x5, 0); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x1 = _mm_xor_si128(x1, x7); + x2 = _mm_xor_si128(x2, x7); + x1 = _mm_xor_si128(x1, x6); + x3 = _mm_xor_si128(x3, x6); + x1 = _mm_xor_si128(x1, x4); + x5 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x1); + x9 = _mm_aesenc_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x7 = _mm_slli_epi32(x2, 31); + x4 = _mm_slli_epi32(x2, 30); + x5 = _mm_slli_epi32(x2, 25); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_xor_si128(x7, x5); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x4 = _mm_bsrli_si128(x7, 4); + x7 = _mm_bslli_si128(x7, 12); + x12 = _mm_aesenc_si128(x12, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x2 = _mm_xor_si128(x2, x7); + x5 = _mm_srli_epi32(x2, 1); + x13 = _mm_aesenc_si128(x13, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x1 = _mm_srli_epi32(x2, 2); + x0 = _mm_srli_epi32(x2, 7); + x14 = _mm_aesenc_si128(x14, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_xor_si128(x5, x1); + x5 = _mm_xor_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x5 = _mm_xor_si128(x5, x4); + x2 = _mm_xor_si128(x2, x5); + x2 = _mm_xor_si128(x2, x3); + zf3 = (word32)rsi; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_update_avx1_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf5 = (word32)rsi; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_decrypt_update_avx1_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_decrypt_update_avx1_aesenc_128_ghash_avx_done: + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_xor_si128(x11, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x14 = _mm_xor_si128(x14, x0); + x15 = _mm_xor_si128(x15, x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + r14 = (word32)((word32)r14 + 0x80); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx1_ghash_128; + } + x6 = x2; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_decrypt_update_avx1_done_128: + rdx = (word32)((word32)r8); + if (((word32)r14) >= ((word32)rdx)) { + goto L_AES_GCM_decrypt_update_avx1_done_dec; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx1_last_block_done; + } +L_AES_GCM_decrypt_update_avx1_last_block_start: + x13 = _mm_loadu_si128((const __m128i*)WC_PR(r11, r14)); + x0 = x5; + x1 = _mm_shuffle_epi8(x13, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x1 = _mm_xor_si128(x1, x6); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_shuffle_epi8(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_epi64, 0))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x9); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x10 = _mm_clmulepi64_si128(x1, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x11 = _mm_clmulepi64_si128(x1, x0, 1); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x12 = _mm_clmulepi64_si128(x1, x0, 0); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x10 = _mm_xor_si128(x10, x11); + x2 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x2 = _mm_xor_si128(x2, x12); + x3 = _mm_xor_si128(x1, x10); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_gcm_mod2_128, 0)); + x11 = _mm_clmulepi64_si128(x2, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x10 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_clmulepi64_si128(x10, x0, 0x10); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x6 = _mm_xor_si128(x10, x3); + zf7 = (word32)rsi; + zf8 = 0xb; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_decrypt_update_avx1_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf9 = (word32)rsi; + zf10 = 0xd; + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_decrypt_update_avx1_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, x9); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_decrypt_update_avx1_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, x9); + x0 = x13; + x8 = _mm_xor_si128(x8, x0); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), x8); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx1_last_block_start; + } +L_AES_GCM_decrypt_update_avx1_last_block_done: +L_AES_GCM_decrypt_update_avx1_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x6); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_decrypt_final_avx1(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res) +{ + word64 rdi, rsi, rax, r10, r11, r9, r8, rbp, rsp, rdx, rcx, r12 = 0, + r13 = 0; + __m128i x0, x1, x5, x6, x7, x8, x9, x10, x11, x12, x13, x15; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rax = (word64)(word32)tbytes; + r10 = (word64)(word32)nbytes; + r11 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + r8 = (word64)(size_t)initCtr; + rbp = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_srli_epi64(x5, 63); + x7 = _mm_slli_epi64(x5, 1); + x8 = _mm_bslli_si128(x8, 8); + x7 = _mm_or_si128(x7, x8); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x7); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x1 = _mm_cvtsi64_si128((long long)rcx); + x0 = _mm_unpacklo_epi64(x0, x1); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_red_avx */ + x8 = _mm_shuffle_epi32(x5, 0x4e); + x9 = _mm_shuffle_epi32(x6, 0x4e); + x10 = _mm_clmulepi64_si128(x6, x5, 0x11); + x7 = _mm_clmulepi64_si128(x6, x5, 0); + x8 = _mm_xor_si128(x8, x5); + x9 = _mm_xor_si128(x9, x6); + x8 = _mm_clmulepi64_si128(x8, x9, 0); + x8 = _mm_xor_si128(x8, x7); + x8 = _mm_xor_si128(x8, x10); + x9 = _mm_bslli_si128(x8, 8); + x8 = _mm_bsrli_si128(x8, 8); + x7 = _mm_xor_si128(x7, x9); + x6 = _mm_xor_si128(x10, x8); + x11 = _mm_slli_epi32(x7, 31); + x12 = _mm_slli_epi32(x7, 30); + x13 = _mm_slli_epi32(x7, 25); + x11 = _mm_xor_si128(x11, x12); + x11 = _mm_xor_si128(x11, x13); + x12 = _mm_bsrli_si128(x11, 4); + x11 = _mm_bslli_si128(x11, 12); + x7 = _mm_xor_si128(x7, x11); + x13 = _mm_srli_epi32(x7, 1); + x9 = _mm_srli_epi32(x7, 2); + x8 = _mm_srli_epi32(x7, 7); + x13 = _mm_xor_si128(x13, x9); + x13 = _mm_xor_si128(x13, x8); + x13 = _mm_xor_si128(x13, x12); + x13 = _mm_xor_si128(x13, x7); + x6 = _mm_xor_si128(x6, x13); + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx1_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x6, x15); + if (((word32)rax) == (0x10)) { + goto L_AES_GCM_decrypt_final_avx1_cmp_tag_16; + } + rsp = (word64)(rsp - 16); + rcx = (word64)(0); + r12 = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_final_avx1_cmp_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsi, + rcx)) & 0xff); + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)((byte)r12 | ( + byte)r13) & 0xff); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)rax)) { + goto L_AES_GCM_decrypt_final_avx1_cmp_tag_loop; + } + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)(((byte)r12) == (0)) & 0xff); + rsp = (word64)(rsp + 16); + rcx = (word64)(0); + goto L_AES_GCM_decrypt_final_avx1_cmp_tag_done; +L_AES_GCM_decrypt_final_avx1_cmp_tag_16: + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_cmpeq_epi8(x0, x1); + rdx = (word32)((word32)_mm_movemask_epi8(x0)); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + r12 = (word32)(0); + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)(((word32)rdx) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_final_avx1_cmp_tag_done: + WC_S32(rbp, 0) = (word32)((word32)r12); +} + +#endif /* WOLFSSL_AESGCM_STREAM */ +#ifdef WOLFSSL_AESGCM_SIV +XALIGNED(16) static const word64 L_aes_gcm_siv_bswap_mask_avx1[] + WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCMSIV_polyval_avx1(unsigned char* s, + const unsigned char* h, const unsigned char* data, unsigned int blocks) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10 = 0; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)h; + rdx = (word64)(size_t)data; + rcx = (word64)(word32)blocks; + + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_siv_bswap_mask_avx1, 0))); + x5 = _mm_clmulepi64_si128(x1, x1, 0); + x6 = _mm_clmulepi64_si128(x1, x1, 0x11); + x10 = _mm_clmulepi64_si128(x1, x1, 0x10); + x11 = _mm_clmulepi64_si128(x1, x1, 1); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x5 = _mm_xor_si128(x5, x11); + x6 = _mm_xor_si128(x6, x10); + x10 = _mm_srli_epi32(x5, 31); + x11 = _mm_srli_epi32(x6, 31); + x5 = _mm_slli_epi32(x5, 1); + x6 = _mm_slli_epi32(x6, 1); + x12 = _mm_bsrli_si128(x10, 12); + x11 = _mm_bslli_si128(x11, 4); + x10 = _mm_bslli_si128(x10, 4); + x5 = _mm_or_si128(x5, x10); + x6 = _mm_or_si128(x6, x11); + x6 = _mm_or_si128(x6, x12); + x10 = _mm_slli_epi32(x5, 31); + x11 = _mm_slli_epi32(x5, 30); + x12 = _mm_slli_epi32(x5, 25); + x10 = _mm_xor_si128(x10, x11); + x10 = _mm_xor_si128(x10, x12); + x11 = _mm_bsrli_si128(x10, 4); + x10 = _mm_bslli_si128(x10, 12); + x5 = _mm_xor_si128(x5, x10); + x7 = _mm_srli_epi32(x5, 1); + x8 = _mm_srli_epi32(x5, 2); + x9 = _mm_srli_epi32(x5, 7); + x7 = _mm_xor_si128(x7, x8); + x7 = _mm_xor_si128(x7, x9); + x7 = _mm_xor_si128(x7, x11); + x5 = _mm_xor_si128(x5, x7); + x2 = _mm_xor_si128(x6, x5); + x5 = _mm_clmulepi64_si128(x2, x1, 0); + x6 = _mm_clmulepi64_si128(x2, x1, 0x11); + x10 = _mm_clmulepi64_si128(x2, x1, 0x10); + x11 = _mm_clmulepi64_si128(x2, x1, 1); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x5 = _mm_xor_si128(x5, x11); + x6 = _mm_xor_si128(x6, x10); + x10 = _mm_srli_epi32(x5, 31); + x11 = _mm_srli_epi32(x6, 31); + x5 = _mm_slli_epi32(x5, 1); + x6 = _mm_slli_epi32(x6, 1); + x12 = _mm_bsrli_si128(x10, 12); + x11 = _mm_bslli_si128(x11, 4); + x10 = _mm_bslli_si128(x10, 4); + x5 = _mm_or_si128(x5, x10); + x6 = _mm_or_si128(x6, x11); + x6 = _mm_or_si128(x6, x12); + x10 = _mm_slli_epi32(x5, 31); + x11 = _mm_slli_epi32(x5, 30); + x12 = _mm_slli_epi32(x5, 25); + x10 = _mm_xor_si128(x10, x11); + x10 = _mm_xor_si128(x10, x12); + x11 = _mm_bsrli_si128(x10, 4); + x10 = _mm_bslli_si128(x10, 12); + x5 = _mm_xor_si128(x5, x10); + x7 = _mm_srli_epi32(x5, 1); + x8 = _mm_srli_epi32(x5, 2); + x9 = _mm_srli_epi32(x5, 7); + x7 = _mm_xor_si128(x7, x8); + x7 = _mm_xor_si128(x7, x9); + x7 = _mm_xor_si128(x7, x11); + x5 = _mm_xor_si128(x5, x7); + x3 = _mm_xor_si128(x6, x5); + x5 = _mm_clmulepi64_si128(x2, x2, 0); + x6 = _mm_clmulepi64_si128(x2, x2, 0x11); + x10 = _mm_clmulepi64_si128(x2, x2, 0x10); + x11 = _mm_clmulepi64_si128(x2, x2, 1); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x5 = _mm_xor_si128(x5, x11); + x6 = _mm_xor_si128(x6, x10); + x10 = _mm_srli_epi32(x5, 31); + x11 = _mm_srli_epi32(x6, 31); + x5 = _mm_slli_epi32(x5, 1); + x6 = _mm_slli_epi32(x6, 1); + x12 = _mm_bsrli_si128(x10, 12); + x11 = _mm_bslli_si128(x11, 4); + x10 = _mm_bslli_si128(x10, 4); + x5 = _mm_or_si128(x5, x10); + x6 = _mm_or_si128(x6, x11); + x6 = _mm_or_si128(x6, x12); + x10 = _mm_slli_epi32(x5, 31); + x11 = _mm_slli_epi32(x5, 30); + x12 = _mm_slli_epi32(x5, 25); + x10 = _mm_xor_si128(x10, x11); + x10 = _mm_xor_si128(x10, x12); + x11 = _mm_bsrli_si128(x10, 4); + x10 = _mm_bslli_si128(x10, 12); + x5 = _mm_xor_si128(x5, x10); + x7 = _mm_srli_epi32(x5, 1); + x8 = _mm_srli_epi32(x5, 2); + x9 = _mm_srli_epi32(x5, 7); + x7 = _mm_xor_si128(x7, x8); + x7 = _mm_xor_si128(x7, x9); + x7 = _mm_xor_si128(x7, x11); + x5 = _mm_xor_si128(x5, x7); + x4 = _mm_xor_si128(x6, x5); + r8 = (word32)((word32)rcx); + r8 = (word32)((word32)r8 << 4); + r9 = (word32)((word32)r8); + r9 = (word32)((word32)r9 & 0xffffffc0); + rax = (word32)(0); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_GCMSIV_polyval_avx1_four_done; + } +L_AES_GCMSIV_polyval_avx1_four: + r10 = (word64)(rdx + rax); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x7 = _mm_xor_si128(x7, x0); + x5 = _mm_clmulepi64_si128(x7, x4, 0); + x6 = _mm_clmulepi64_si128(x7, x4, 0x11); + x10 = _mm_clmulepi64_si128(x7, x4, 0x10); + x11 = _mm_clmulepi64_si128(x7, x4, 1); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x5 = _mm_xor_si128(x5, x11); + x6 = _mm_xor_si128(x6, x10); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_clmulepi64_si128(x7, x3, 0); + x9 = _mm_clmulepi64_si128(x7, x3, 0x11); + x10 = _mm_clmulepi64_si128(x7, x3, 0x10); + x11 = _mm_clmulepi64_si128(x7, x3, 1); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x8 = _mm_xor_si128(x8, x11); + x9 = _mm_xor_si128(x9, x10); + x5 = _mm_xor_si128(x5, x8); + x6 = _mm_xor_si128(x6, x9); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_clmulepi64_si128(x7, x2, 0); + x9 = _mm_clmulepi64_si128(x7, x2, 0x11); + x10 = _mm_clmulepi64_si128(x7, x2, 0x10); + x11 = _mm_clmulepi64_si128(x7, x2, 1); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x8 = _mm_xor_si128(x8, x11); + x9 = _mm_xor_si128(x9, x10); + x5 = _mm_xor_si128(x5, x8); + x6 = _mm_xor_si128(x6, x9); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_clmulepi64_si128(x7, x1, 0); + x9 = _mm_clmulepi64_si128(x7, x1, 0x11); + x10 = _mm_clmulepi64_si128(x7, x1, 0x10); + x11 = _mm_clmulepi64_si128(x7, x1, 1); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x8 = _mm_xor_si128(x8, x11); + x9 = _mm_xor_si128(x9, x10); + x5 = _mm_xor_si128(x5, x8); + x6 = _mm_xor_si128(x6, x9); + x10 = _mm_srli_epi32(x5, 31); + x11 = _mm_srli_epi32(x6, 31); + x5 = _mm_slli_epi32(x5, 1); + x6 = _mm_slli_epi32(x6, 1); + x12 = _mm_bsrli_si128(x10, 12); + x11 = _mm_bslli_si128(x11, 4); + x10 = _mm_bslli_si128(x10, 4); + x5 = _mm_or_si128(x5, x10); + x6 = _mm_or_si128(x6, x11); + x6 = _mm_or_si128(x6, x12); + x10 = _mm_slli_epi32(x5, 31); + x11 = _mm_slli_epi32(x5, 30); + x12 = _mm_slli_epi32(x5, 25); + x10 = _mm_xor_si128(x10, x11); + x10 = _mm_xor_si128(x10, x12); + x11 = _mm_bsrli_si128(x10, 4); + x10 = _mm_bslli_si128(x10, 12); + x5 = _mm_xor_si128(x5, x10); + x7 = _mm_srli_epi32(x5, 1); + x8 = _mm_srli_epi32(x5, 2); + x9 = _mm_srli_epi32(x5, 7); + x7 = _mm_xor_si128(x7, x8); + x7 = _mm_xor_si128(x7, x9); + x7 = _mm_xor_si128(x7, x11); + x5 = _mm_xor_si128(x5, x7); + x0 = _mm_xor_si128(x6, x5); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_GCMSIV_polyval_avx1_four; + } +L_AES_GCMSIV_polyval_avx1_four_done: +L_AES_GCMSIV_polyval_avx1_rem: + if (((word32)rax) >= ((word32)r8)) { + goto L_AES_GCMSIV_polyval_avx1_done; + } + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, rax))); + x5 = _mm_clmulepi64_si128(x0, x1, 0); + x6 = _mm_clmulepi64_si128(x0, x1, 0x11); + x10 = _mm_clmulepi64_si128(x0, x1, 0x10); + x11 = _mm_clmulepi64_si128(x0, x1, 1); + x10 = _mm_xor_si128(x10, x11); + x11 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x5 = _mm_xor_si128(x5, x11); + x6 = _mm_xor_si128(x6, x10); + x10 = _mm_srli_epi32(x5, 31); + x11 = _mm_srli_epi32(x6, 31); + x5 = _mm_slli_epi32(x5, 1); + x6 = _mm_slli_epi32(x6, 1); + x12 = _mm_bsrli_si128(x10, 12); + x11 = _mm_bslli_si128(x11, 4); + x10 = _mm_bslli_si128(x10, 4); + x5 = _mm_or_si128(x5, x10); + x6 = _mm_or_si128(x6, x11); + x6 = _mm_or_si128(x6, x12); + x10 = _mm_slli_epi32(x5, 31); + x11 = _mm_slli_epi32(x5, 30); + x12 = _mm_slli_epi32(x5, 25); + x10 = _mm_xor_si128(x10, x11); + x10 = _mm_xor_si128(x10, x12); + x11 = _mm_bsrli_si128(x10, 4); + x10 = _mm_bslli_si128(x10, 12); + x5 = _mm_xor_si128(x5, x10); + x7 = _mm_srli_epi32(x5, 1); + x8 = _mm_srli_epi32(x5, 2); + x9 = _mm_srli_epi32(x5, 7); + x7 = _mm_xor_si128(x7, x8); + x7 = _mm_xor_si128(x7, x9); + x7 = _mm_xor_si128(x7, x11); + x5 = _mm_xor_si128(x5, x7); + x0 = _mm_xor_si128(x6, x5); + rax = (word32)((word32)rax + 0x10); + goto L_AES_GCMSIV_polyval_avx1_rem; +L_AES_GCMSIV_polyval_avx1_done: + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_siv_bswap_mask_avx1, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); +} + +XALIGNED(16) static const word64 L_aes_gcmsiv_ctr_avx1_one[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_GCMSIV_ctr_avx1(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11 = 0, r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7, x8; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + r9 = (word64)(size_t)ctr; + + x8 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_gcmsiv_ctr_avx1_one, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x40; + r10 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_GCMSIV_ctr_avx1_done_64; + } + r10 = (word32)((word32)r10 & 0xffffffc0); +L_AES_GCMSIV_ctr_avx1_enc_64: + /* 64 bytes of input */ + /* siv_ctr_enc_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + x0 = x7; + x7 = _mm_add_epi32(x7, x8); + x1 = x7; + x7 = _mm_add_epi32(x7, x8); + x2 = x7; + x7 = _mm_add_epi32(x7, x8); + x3 = x7; + x7 = _mm_add_epi32(x7, x8); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_xor_si128(x0, x4); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x4); + x3 = _mm_xor_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf3 = (word32)r8; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCMSIV_ctr_avx1_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf5 = (word32)r8; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCMSIV_ctr_avx1_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_GCMSIV_ctr_avx1_64_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x4); + x1 = _mm_aesenclast_si128(x1, x4); + x2 = _mm_aesenclast_si128(x2, x4); + x3 = _mm_aesenclast_si128(x3, x4); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x1 = _mm_xor_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(r11, 16))); + x2 = _mm_xor_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(r11, 32))); + x3 = _mm_xor_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(r11, 48))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r12, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r12, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r12, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_avx1_enc_64; + } +L_AES_GCMSIV_ctr_avx1_done_64: + zf7 = (word32)rax; + zf8 = (word32)rdx; + r10 = (word32)((word32)rdx); + if ((zf7) == (zf8)) { + goto L_AES_GCMSIV_ctr_avx1_done_enc; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_GCMSIV_ctr_avx1_enc_16: + /* 16 bytes of input */ + x0 = x7; + x7 = _mm_add_epi32(x7, x8); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf9 = (word32)r8; + zf10 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCMSIV_ctr_avx1_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf11 = (word32)r8; + zf12 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCMSIV_ctr_avx1_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_GCMSIV_ctr_avx1_16_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r11 = (word64)(rdi + rax); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_avx1_enc_16; + } +L_AES_GCMSIV_ctr_avx1_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x7); +} + +#endif /* WOLFSSL_AESGCM_SIV */ +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +XALIGNED(16) static const word64 L_GCM_generate_m0_avx2_rev8[] + WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_GCM_generate_m0_avx2_mod2_128[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0xe100000000000000ULL, +}; + +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL void GCM_generate_m0_avx2(const unsigned char* h, + unsigned char* m) +{ + word64 rdi, rsi; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10; + + rdi = (word64)(size_t)h; + rsi = (word64)(size_t)m; + + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_GCM_generate_m0_avx2_rev8, 0)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(L_GCM_generate_m0_avx2_mod2_128, + 0)); + x8 = _mm_setzero_si128(); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x8); + x8 = x0; + x0 = _mm_shuffle_epi8(x0, x9); + x5 = _mm_slli_epi64(x0, 63); + x4 = _mm_srli_epi64(x0, 1); + x1 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x1 = _mm_shuffle_epi32(x1, 0xff); + x4 = _mm_or_si128(x4, x5); + x1 = _mm_srai_epi32(x1, 31); + x1 = _mm_and_si128(x1, x10); + x1 = _mm_xor_si128(x1, x4); + x5 = _mm_slli_epi64(x1, 63); + x4 = _mm_srli_epi64(x1, 1); + x2 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x2 = _mm_shuffle_epi32(x2, 0xff); + x4 = _mm_or_si128(x4, x5); + x2 = _mm_srai_epi32(x2, 31); + x2 = _mm_and_si128(x2, x10); + x2 = _mm_xor_si128(x2, x4); + x5 = _mm_slli_epi64(x2, 63); + x4 = _mm_srli_epi64(x2, 1); + x3 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x3 = _mm_shuffle_epi32(x3, 0xff); + x4 = _mm_or_si128(x4, x5); + x3 = _mm_srai_epi32(x3, 31); + x3 = _mm_and_si128(x3, x10); + x3 = _mm_xor_si128(x3, x4); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x8 = _mm_xor_si128(x3, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x3); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x1); + x4 = _mm_xor_si128(x3, x1); + x5 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x8, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 128), x0); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x3, x0); + x6 = _mm_xor_si128(x2, x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 144), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 160), x6); + x6 = _mm_xor_si128(x3, x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 176), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 192), x1); + x4 = _mm_xor_si128(x3, x1); + x5 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x8, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 208), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 224), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 240), x6); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 256), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 272), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 288), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 304), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 320), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 336), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 352), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 368), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 384), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 400), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 416), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 432), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 240)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 448), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 464), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 480), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 496), x3); +} + +XALIGNED(16) static const word64 L_avx2_aes_gcm_one[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000001ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_two[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_three[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000003ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_four[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_five[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000005ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_six[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000006ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_seven[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000007ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_eight[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000008ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_bswap_one[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0100000000000000ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_bswap_epi64[] WC_X64I_UNUSED = { + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_bswap_mask[] WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_avx2_aes_gcm_mod2_128[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0xc200000000000000ULL, +}; + +WC_X64I_TARGET("aes,pclmul,avx2") +WOLFSSL_LOCAL void AES_GCM_encrypt_avx2(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr) +{ + word64 rdi, r8, r12, rax, r15, r10, r11, rbx, r14, rsi, r9, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6, x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[24]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + + rdi = (word64)(size_t)in; + r8 = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r15 = (word64)(size_t)tag; + r10 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r14 = (word64)(word64)tbytes; + rsi = (word64)(size_t)key; + r9 = (word64)(word64)nr; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 184); + x4 = _mm_setzero_si128(); + x6 = _mm_setzero_si128(); + rdx = (word32)((word32)rbx); + if (((word32)rdx) == (0xc)) { + goto L_AES_GCM_encrypt_avx2_iv_12; + } + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144))); + zf1 = (word32)r9; + zf2 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176))); + zf3 = (word32)r9; + zf4 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_encrypt_avx2_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x0); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf5 = (word32)rdx; + zf6 = 0; + rcx = (word64)(0); + if ((zf5) == (zf6)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_encrypt_avx2_calc_iv_16_loop: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rax, rcx)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_done; + } +L_AES_GCM_encrypt_avx2_calc_iv_lt16: + x0 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_avx2_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_loop; + } + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); +L_AES_GCM_encrypt_avx2_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x15 = _mm_xor_si128(x15, x4); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 144))); + zf7 = (word32)r9; + zf8 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last; + } + x15 = _mm_aesenc_si128(x15, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 176))); + zf9 = (word32)r9; + zf10 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last; + } + x15 = _mm_aesenc_si128(x15, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_encrypt_avx2_calc_iv_2_aesenc_avx_last: + x15 = _mm_aesenclast_si128(x15, x0); + goto L_AES_GCM_encrypt_avx2_iv_done; +L_AES_GCM_encrypt_avx2_iv_12: + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_one, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x4 = _mm_blend_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)), 7); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x15 = _mm_xor_si128(x4, x5); + x5 = _mm_aesenc_si128(x5, x7); + x15 = _mm_aesenc_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + zf11 = (word32)r9; + zf12 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + zf13 = (word32)r9; + zf14 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_avx2_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_encrypt_avx2_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x0); + x15 = _mm_aesenclast_si128(x15, x0); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); +L_AES_GCM_encrypt_avx2_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_encrypt_avx2_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x80)) { + goto L_AES_GCM_encrypt_avx2_calc_aad_after_agg; + } + r13 = (word32)((word32)rdx); + r13 = (word32)((word32)r13 & 0xffffffc0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 168), x5); + x1 = _mm_srli_epi64(x5, 63); + x0 = _mm_slli_epi64(x5, 1); + x1 = _mm_bslli_si128(x1, 8); + x0 = _mm_or_si128(x0, x1); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x0); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + /* H ^ 1 and H ^ 2 */ + x9 = _mm_clmulepi64_si128(x5, x5, 0); + x10 = _mm_clmulepi64_si128(x5, x5, 0x11); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x0 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 and H ^ 4 */ + x11 = _mm_clmulepi64_si128(x0, x5, 0x10); + x10 = _mm_clmulepi64_si128(x0, x5, 1); + x9 = _mm_clmulepi64_si128(x0, x5, 0); + x12 = _mm_clmulepi64_si128(x0, x5, 0x11); + x13 = _mm_clmulepi64_si128(x0, x0, 0); + x14 = _mm_clmulepi64_si128(x0, x0, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x2 = _mm_xor_si128(x13, x14); + x1 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x2); + /* H ^ 5 and H ^ 6 */ + x11 = _mm_clmulepi64_si128(x1, x0, 0x10); + x10 = _mm_clmulepi64_si128(x1, x0, 1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x13 = _mm_clmulepi64_si128(x1, x1, 0); + x14 = _mm_clmulepi64_si128(x1, x1, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x0); + /* H ^ 7 and H ^ 8 */ + x11 = _mm_clmulepi64_si128(x2, x1, 0x10); + x10 = _mm_clmulepi64_si128(x2, x1, 1); + x9 = _mm_clmulepi64_si128(x2, x1, 0); + x12 = _mm_clmulepi64_si128(x2, x1, 0x11); + x13 = _mm_clmulepi64_si128(x2, x2, 0); + x14 = _mm_clmulepi64_si128(x2, x2, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x0); +L_AES_GCM_encrypt_avx2_calc_aad_agg: + /* 64 bytes of AAD */ + rbx = (word64)(r12 + rcx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 0)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 16)); + x9 = _mm_shuffle_epi8(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 32)); + x10 = _mm_shuffle_epi8(x10, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 48)); + x11 = _mm_shuffle_epi8(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x8 = _mm_xor_si128(x8, x6); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x13 = _mm_clmulepi64_si128(x7, x8, 0x10); + x1 = _mm_clmulepi64_si128(x7, x8, 1); + x12 = _mm_clmulepi64_si128(x7, x8, 0); + x6 = _mm_clmulepi64_si128(x7, x8, 0x11); + x13 = _mm_xor_si128(x13, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x2 = _mm_clmulepi64_si128(x7, x9, 0x10); + x1 = _mm_clmulepi64_si128(x7, x9, 1); + x0 = _mm_clmulepi64_si128(x7, x9, 0); + x3 = _mm_clmulepi64_si128(x7, x9, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x13 = _mm_xor_si128(x13, x2); + x12 = _mm_xor_si128(x12, x0); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x2 = _mm_clmulepi64_si128(x7, x10, 0x10); + x1 = _mm_clmulepi64_si128(x7, x10, 1); + x0 = _mm_clmulepi64_si128(x7, x10, 0); + x3 = _mm_clmulepi64_si128(x7, x10, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x13 = _mm_xor_si128(x13, x2); + x12 = _mm_xor_si128(x12, x0); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x2 = _mm_clmulepi64_si128(x7, x11, 0x10); + x1 = _mm_clmulepi64_si128(x7, x11, 1); + x0 = _mm_clmulepi64_si128(x7, x11, 0); + x3 = _mm_clmulepi64_si128(x7, x11, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x13 = _mm_xor_si128(x13, x2); + x12 = _mm_xor_si128(x12, x0); + x7 = _mm_bslli_si128(x13, 8); + x13 = _mm_bsrli_si128(x13, 8); + x12 = _mm_xor_si128(x12, x7); + x6 = _mm_xor_si128(x6, x13); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x12, x2, 0x10); + x1 = _mm_shuffle_epi32(x12, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_xor_si128(x6, x1); + rcx = (word32)((word32)rcx + 0x40); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx2_calc_aad_agg; + } + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 168)); +L_AES_GCM_encrypt_avx2_calc_aad_after_agg: + rdx = (word32)((word32)r11); + rdx = (word32)((word32)rdx & 0xfffffff0); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx2_calc_aad_partial; + } +L_AES_GCM_encrypt_avx2_calc_aad_16_loop: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r12, rcx)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x6, 0x10); + x1 = _mm_clmulepi64_si128(x5, x6, 1); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + x3 = _mm_clmulepi64_si128(x5, x6, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x6 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x6, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_xor_si128(x6, x1); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx2_calc_aad_16_loop; + } +L_AES_GCM_encrypt_avx2_calc_aad_partial: + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx2_calc_aad_done; + } + x0 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_avx2_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx2_calc_aad_loop; + } + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x6, 0x10); + x1 = _mm_clmulepi64_si128(x5, x6, 1); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + x3 = _mm_clmulepi64_si128(x5, x6, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x6 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x6, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_xor_si128(x6, x1); +L_AES_GCM_encrypt_avx2_calc_aad_done: + /* Calculate counter and H */ + x1 = _mm_srli_epi64(x5, 63); + x0 = _mm_slli_epi64(x5, 1); + x1 = _mm_bslli_si128(x1, 8); + x0 = _mm_or_si128(x0, x1); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x5 = _mm_xor_si128(x5, x0); + rbx = (word32)(0); + zf15 = (word32)r10; + zf16 = 0x80; + r13 = (word32)((word32)r10); + if ((zf15) < (zf16)) { + goto L_AES_GCM_encrypt_avx2_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x15); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + /* H ^ 1 and H ^ 2 */ + x9 = _mm_clmulepi64_si128(x5, x5, 0); + x10 = _mm_clmulepi64_si128(x5, x5, 0x11); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x0 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 and H ^ 4 */ + x11 = _mm_clmulepi64_si128(x0, x5, 0x10); + x10 = _mm_clmulepi64_si128(x0, x5, 1); + x9 = _mm_clmulepi64_si128(x0, x5, 0); + x12 = _mm_clmulepi64_si128(x0, x5, 0x11); + x13 = _mm_clmulepi64_si128(x0, x0, 0); + x14 = _mm_clmulepi64_si128(x0, x0, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x2 = _mm_xor_si128(x13, x14); + x1 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x2); + /* H ^ 5 and H ^ 6 */ + x11 = _mm_clmulepi64_si128(x1, x0, 0x10); + x10 = _mm_clmulepi64_si128(x1, x0, 1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x13 = _mm_clmulepi64_si128(x1, x1, 0); + x14 = _mm_clmulepi64_si128(x1, x1, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x0); + /* H ^ 7 and H ^ 8 */ + x11 = _mm_clmulepi64_si128(x2, x1, 0x10); + x10 = _mm_clmulepi64_si128(x2, x1, 1); + x9 = _mm_clmulepi64_si128(x2, x1, 0); + x12 = _mm_clmulepi64_si128(x2, x1, 0x11); + x13 = _mm_clmulepi64_si128(x2, x2, 0); + x14 = _mm_clmulepi64_si128(x2, x2, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x0); + /* First 128 bytes of input */ + /* aesenc_128 */ + /* aesenc_ctr */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_epi64, 0)); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x8 = _mm_shuffle_epi8(x0, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_two, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_three, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_four, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_five, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_six, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_seven, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_eight, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + /* aesenc_xor */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf17 = (word32)r9; + zf18 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_encrypt_avx2_aesenc_128_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf19 = (word32)r9; + zf20 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_encrypt_avx2_aesenc_128_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_encrypt_avx2_aesenc_128_enc_done: + /* aesenc_last */ + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(r8, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(r8, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(r8, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + _mm_storeu_si128((__m128i*)WC_PW(r8, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(r8, 80), x13); + _mm_storeu_si128((__m128i*)WC_PW(r8, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(r8, 112), x15); + zf21 = (word32)r13; + zf22 = 0x80; + rbx = (word32)(0x80); + if ((zf21) <= (zf22)) { + goto L_AES_GCM_encrypt_avx2_end_128; + } + /* More 128 bytes of input */ +L_AES_GCM_encrypt_avx2_ghash_128: + /* aesenc_128_ghash */ + rcx = (word64)(rdi + rbx); + rdx = (word64)(r8 + rbx); + /* aesenc_ctr */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_epi64, 0)); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x8 = _mm_shuffle_epi8(x0, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_two, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_three, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_four, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_five, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_six, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_seven, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_eight, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + /* aesenc_xor */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + /* aesenc_pclmul_1 */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -128)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x1 = _mm_xor_si128(x1, x6); + x5 = _mm_clmulepi64_si128(x1, x2, 0x10); + x3 = _mm_clmulepi64_si128(x1, x2, 1); + x6 = _mm_clmulepi64_si128(x1, x2, 0); + x7 = _mm_clmulepi64_si128(x1, x2, 0x11); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_2 */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x3); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_l */ + x5 = _mm_xor_si128(x5, x2); + x6 = _mm_xor_si128(x6, x4); + x5 = _mm_xor_si128(x5, x3); + x1 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x8 = _mm_aesenc_si128(x8, x4); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_xor_si128(x7, x5); + x3 = _mm_clmulepi64_si128(x6, x0, 0x10); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x6 = _mm_xor_si128(x6, x3); + x3 = _mm_clmulepi64_si128(x6, x0, 0x10); + x12 = _mm_aesenc_si128(x12, x4); + x13 = _mm_aesenc_si128(x13, x4); + x14 = _mm_aesenc_si128(x14, x4); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x6 = _mm_xor_si128(x6, x3); + x6 = _mm_xor_si128(x6, x7); + x15 = _mm_aesenc_si128(x15, x4); + zf23 = (word32)r9; + zf24 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf25 = (word32)r9; + zf26 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_encrypt_avx2_aesenc_128_ghash_avx_done: + /* aesenc_last */ + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + /* aesenc_128_ghash - end */ + rbx = (word32)((word32)rbx + 0x80); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx2_ghash_128; + } +L_AES_GCM_encrypt_avx2_end_128: + x4 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_mask, 0)); + x8 = _mm_shuffle_epi8(x8, x4); + x9 = _mm_shuffle_epi8(x9, x4); + x10 = _mm_shuffle_epi8(x10, x4); + x11 = _mm_shuffle_epi8(x11, x4); + x12 = _mm_shuffle_epi8(x12, x4); + x13 = _mm_shuffle_epi8(x13, x4); + x14 = _mm_shuffle_epi8(x14, x4); + x15 = _mm_shuffle_epi8(x15, x4); + x8 = _mm_xor_si128(x8, x6); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x5 = _mm_clmulepi64_si128(x7, x15, 0x10); + x1 = _mm_clmulepi64_si128(x7, x15, 1); + x4 = _mm_clmulepi64_si128(x7, x15, 0); + x6 = _mm_clmulepi64_si128(x7, x15, 0x11); + x5 = _mm_xor_si128(x5, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x2 = _mm_clmulepi64_si128(x7, x14, 0x10); + x1 = _mm_clmulepi64_si128(x7, x14, 1); + x0 = _mm_clmulepi64_si128(x7, x14, 0); + x3 = _mm_clmulepi64_si128(x7, x14, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x2 = _mm_clmulepi64_si128(x15, x13, 0x10); + x1 = _mm_clmulepi64_si128(x15, x13, 1); + x0 = _mm_clmulepi64_si128(x15, x13, 0); + x3 = _mm_clmulepi64_si128(x15, x13, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x2 = _mm_clmulepi64_si128(x7, x12, 0x10); + x1 = _mm_clmulepi64_si128(x7, x12, 1); + x0 = _mm_clmulepi64_si128(x7, x12, 0); + x3 = _mm_clmulepi64_si128(x7, x12, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x2 = _mm_clmulepi64_si128(x15, x11, 0x10); + x1 = _mm_clmulepi64_si128(x15, x11, 1); + x0 = _mm_clmulepi64_si128(x15, x11, 0); + x3 = _mm_clmulepi64_si128(x15, x11, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x2 = _mm_clmulepi64_si128(x7, x10, 0x10); + x1 = _mm_clmulepi64_si128(x7, x10, 1); + x0 = _mm_clmulepi64_si128(x7, x10, 0); + x3 = _mm_clmulepi64_si128(x7, x10, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x2 = _mm_clmulepi64_si128(x15, x9, 0x10); + x1 = _mm_clmulepi64_si128(x15, x9, 1); + x0 = _mm_clmulepi64_si128(x15, x9, 0); + x3 = _mm_clmulepi64_si128(x15, x9, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x2 = _mm_clmulepi64_si128(x7, x8, 0x10); + x1 = _mm_clmulepi64_si128(x7, x8, 1); + x0 = _mm_clmulepi64_si128(x7, x8, 0); + x3 = _mm_clmulepi64_si128(x7, x8, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x7 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x4 = _mm_xor_si128(x4, x7); + x6 = _mm_xor_si128(x6, x5); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x4, x2, 0x10); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_xor_si128(x6, x1); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); +L_AES_GCM_encrypt_avx2_done_128: + if (((word32)rbx) == ((word32)r10)) { + goto L_AES_GCM_encrypt_avx2_done_enc; + } + r13 = (word32)((word32)r10); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx2_last_block_done; + } + /* aesenc_block */ + x1 = x4; + x0 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x0 = _mm_aesenc_si128(x0, x2); + x4 = x1; + zf27 = (word32)r9; + zf28 = 0xb; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_encrypt_avx2_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_aesenc_si128(x0, x2); + zf29 = (word32)r9; + zf30 = 0xd; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_encrypt_avx2_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x0 = _mm_aesenc_si128(x0, x2); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_encrypt_avx2_aesenc_block_last: + x0 = _mm_aesenclast_si128(x0, x1); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rbx)); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(r8, rbx), x0); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x0); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx2_last_block_ghash; + } +L_AES_GCM_encrypt_avx2_last_block_start: + x12 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rbx)); + x11 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + /* aesenc_gfmul_sb */ + x2 = _mm_clmulepi64_si128(x6, x5, 1); + x3 = _mm_clmulepi64_si128(x6, x5, 0x10); + x1 = _mm_clmulepi64_si128(x6, x5, 0); + x8 = _mm_clmulepi64_si128(x6, x5, 0x11); + x11 = _mm_xor_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 16))); + x3 = _mm_xor_si128(x3, x2); + x2 = _mm_bslli_si128(x3, 8); + x3 = _mm_bsrli_si128(x3, 8); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 32))); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_clmulepi64_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 48))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 64))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 80))); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_clmulepi64_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 96))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 112))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 128))); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 144))); + x8 = _mm_xor_si128(x8, x3); + x2 = _mm_xor_si128(x2, x8); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)((word32)r9) < (sword32)(0xb)) { + goto L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last; + } + x11 = _mm_aesenc_si128(x11, x0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 176))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)((word32)r9) < (sword32)(0xd)) { + goto L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last; + } + x11 = _mm_aesenc_si128(x11, x0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_encrypt_avx2_aesenc_gfmul_sb_last: + x11 = _mm_aesenclast_si128(x11, x0); + x6 = _mm_xor_si128(x2, x1); + x11 = _mm_xor_si128(x11, x12); + _mm_storeu_si128((__m128i*)WC_PW(r8, rbx), x11); + x11 = _mm_shuffle_epi8(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x11); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx2_last_block_start; + } +L_AES_GCM_encrypt_avx2_last_block_ghash: + /* ghash_gfmul_red */ + x10 = _mm_clmulepi64_si128(x6, x5, 0x10); + x9 = _mm_clmulepi64_si128(x6, x5, 1); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + x10 = _mm_xor_si128(x10, x9); + x9 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x9 = _mm_xor_si128(x9, x8); + x6 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_clmulepi64_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x8 = _mm_clmulepi64_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x6 = _mm_xor_si128(x6, x10); + x6 = _mm_xor_si128(x6, x9); + x6 = _mm_xor_si128(x6, x8); +L_AES_GCM_encrypt_avx2_last_block_done: + rcx = (word32)((word32)r10); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_encrypt_avx2_done_enc; + } + /* aesenc_last15_enc */ + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x4 = _mm_xor_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144))); + zf31 = (word32)r9; + zf32 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x0); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176))); + zf33 = (word32)r9; + zf34 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x0); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_aesenc_avx_last: + x4 = _mm_aesenclast_si128(x4, x0); + rcx = (word32)(0); + x0 = _mm_setzero_si128(); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); +L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(rsp, rcx + 16) = (byte)((byte)r13); + WC_S8(r8, rbx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx2_aesenc_last15_enc_avx_loop; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x4); + /* ghash_gfmul_red */ + x2 = _mm_clmulepi64_si128(x6, x5, 0x10); + x1 = _mm_clmulepi64_si128(x6, x5, 1); + x0 = _mm_clmulepi64_si128(x6, x5, 0); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_clmulepi64_si128(x6, x5, 0x11); + x0 = _mm_clmulepi64_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x6 = _mm_xor_si128(x6, x2); + x6 = _mm_xor_si128(x6, x1); + x6 = _mm_xor_si128(x6, x0); +L_AES_GCM_encrypt_avx2_done_enc: + /* calc_tag */ + r10 = (word64)(r10 << 3); + r11 = (word64)(r11 << 3); + x0 = _mm_cvtsi64_si128((long long)r10); + x1 = _mm_cvtsi64_si128((long long)r11); + x0 = _mm_unpacklo_epi64(x0, x1); + x0 = _mm_xor_si128(x0, x6); + /* ghash_gfmul_red */ + x4 = _mm_clmulepi64_si128(x0, x5, 0x10); + x3 = _mm_clmulepi64_si128(x0, x5, 1); + x2 = _mm_clmulepi64_si128(x0, x5, 0); + x4 = _mm_xor_si128(x4, x3); + x3 = _mm_bslli_si128(x4, 8); + x4 = _mm_bsrli_si128(x4, 8); + x3 = _mm_xor_si128(x3, x2); + x0 = _mm_clmulepi64_si128(x0, x5, 0x11); + x2 = _mm_clmulepi64_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x3 = _mm_shuffle_epi32(x3, 0x4e); + x3 = _mm_xor_si128(x3, x2); + x2 = _mm_clmulepi64_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x3 = _mm_shuffle_epi32(x3, 0x4e); + x0 = _mm_xor_si128(x0, x4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x15); + /* store_tag */ + if (((word32)r14) == (0x10)) { + goto L_AES_GCM_encrypt_avx2_store_tag_16; + } + rcx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_avx2_store_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + WC_S8(r15, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)r14)) { + goto L_AES_GCM_encrypt_avx2_store_tag_loop; + } + goto L_AES_GCM_encrypt_avx2_store_tag_done; +L_AES_GCM_encrypt_avx2_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(r15, 0), x0); +L_AES_GCM_encrypt_avx2_store_tag_done: + ; +} + +WC_X64I_TARGET("aes,pclmul,avx2") +WOLFSSL_LOCAL void AES_GCM_decrypt_avx2(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res) +{ + word64 rdi, r8, r12, rax, r14, r10, r11, rbx, r15, rsi, r9, rbp, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6, x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[24]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + + rdi = (word64)(size_t)in; + r8 = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r14 = (word64)(size_t)tag; + r10 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r15 = (word64)(word64)tbytes; + rsi = (word64)(size_t)key; + r9 = (word64)(word64)nr; + rbp = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 184); + x4 = _mm_setzero_si128(); + x6 = _mm_setzero_si128(); + rdx = (word32)((word32)rbx); + if (((word32)rdx) == (0xc)) { + goto L_AES_GCM_decrypt_avx2_iv_12; + } + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144))); + zf1 = (word32)r9; + zf2 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176))); + zf3 = (word32)r9; + zf4 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_decrypt_avx2_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x0); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf5 = (word32)rdx; + zf6 = 0; + rcx = (word64)(0); + if ((zf5) == (zf6)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_decrypt_avx2_calc_iv_16_loop: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rax, rcx)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_done; + } +L_AES_GCM_decrypt_avx2_calc_iv_lt16: + x0 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_avx2_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_loop; + } + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); +L_AES_GCM_decrypt_avx2_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x15 = _mm_xor_si128(x15, x4); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 16))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 32))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 48))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 64))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 80))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 96))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 112))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 128))); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 144))); + zf7 = (word32)r9; + zf8 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last; + } + x15 = _mm_aesenc_si128(x15, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 176))); + zf9 = (word32)r9; + zf10 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last; + } + x15 = _mm_aesenc_si128(x15, x0); + x15 = _mm_aesenc_si128(x15, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_decrypt_avx2_calc_iv_2_aesenc_avx_last: + x15 = _mm_aesenclast_si128(x15, x0); + goto L_AES_GCM_decrypt_avx2_iv_done; +L_AES_GCM_decrypt_avx2_iv_12: + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_one, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x4 = _mm_blend_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)), 7); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x15 = _mm_xor_si128(x4, x5); + x5 = _mm_aesenc_si128(x5, x7); + x15 = _mm_aesenc_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + zf11 = (word32)r9; + zf12 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + zf13 = (word32)r9; + zf14 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_decrypt_avx2_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x5 = _mm_aesenc_si128(x5, x0); + x15 = _mm_aesenc_si128(x15, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_decrypt_avx2_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x0); + x15 = _mm_aesenclast_si128(x15, x0); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); +L_AES_GCM_decrypt_avx2_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_decrypt_avx2_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x80)) { + goto L_AES_GCM_decrypt_avx2_calc_aad_after_agg; + } + r13 = (word32)((word32)rdx); + r13 = (word32)((word32)r13 & 0xffffffc0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 168), x5); + x1 = _mm_srli_epi64(x5, 63); + x0 = _mm_slli_epi64(x5, 1); + x1 = _mm_bslli_si128(x1, 8); + x0 = _mm_or_si128(x0, x1); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x0); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + /* H ^ 1 and H ^ 2 */ + x9 = _mm_clmulepi64_si128(x5, x5, 0); + x10 = _mm_clmulepi64_si128(x5, x5, 0x11); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x0 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 and H ^ 4 */ + x11 = _mm_clmulepi64_si128(x0, x5, 0x10); + x10 = _mm_clmulepi64_si128(x0, x5, 1); + x9 = _mm_clmulepi64_si128(x0, x5, 0); + x12 = _mm_clmulepi64_si128(x0, x5, 0x11); + x13 = _mm_clmulepi64_si128(x0, x0, 0); + x14 = _mm_clmulepi64_si128(x0, x0, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x2 = _mm_xor_si128(x13, x14); + x1 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x2); + /* H ^ 5 and H ^ 6 */ + x11 = _mm_clmulepi64_si128(x1, x0, 0x10); + x10 = _mm_clmulepi64_si128(x1, x0, 1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x13 = _mm_clmulepi64_si128(x1, x1, 0); + x14 = _mm_clmulepi64_si128(x1, x1, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x0); + /* H ^ 7 and H ^ 8 */ + x11 = _mm_clmulepi64_si128(x2, x1, 0x10); + x10 = _mm_clmulepi64_si128(x2, x1, 1); + x9 = _mm_clmulepi64_si128(x2, x1, 0); + x12 = _mm_clmulepi64_si128(x2, x1, 0x11); + x13 = _mm_clmulepi64_si128(x2, x2, 0); + x14 = _mm_clmulepi64_si128(x2, x2, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x0); +L_AES_GCM_decrypt_avx2_calc_aad_agg: + /* 64 bytes of AAD */ + rbx = (word64)(r12 + rcx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 0)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 16)); + x9 = _mm_shuffle_epi8(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 32)); + x10 = _mm_shuffle_epi8(x10, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 48)); + x11 = _mm_shuffle_epi8(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x8 = _mm_xor_si128(x8, x6); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x13 = _mm_clmulepi64_si128(x7, x8, 0x10); + x1 = _mm_clmulepi64_si128(x7, x8, 1); + x12 = _mm_clmulepi64_si128(x7, x8, 0); + x6 = _mm_clmulepi64_si128(x7, x8, 0x11); + x13 = _mm_xor_si128(x13, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x2 = _mm_clmulepi64_si128(x7, x9, 0x10); + x1 = _mm_clmulepi64_si128(x7, x9, 1); + x0 = _mm_clmulepi64_si128(x7, x9, 0); + x3 = _mm_clmulepi64_si128(x7, x9, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x13 = _mm_xor_si128(x13, x2); + x12 = _mm_xor_si128(x12, x0); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x2 = _mm_clmulepi64_si128(x7, x10, 0x10); + x1 = _mm_clmulepi64_si128(x7, x10, 1); + x0 = _mm_clmulepi64_si128(x7, x10, 0); + x3 = _mm_clmulepi64_si128(x7, x10, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x13 = _mm_xor_si128(x13, x2); + x12 = _mm_xor_si128(x12, x0); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x2 = _mm_clmulepi64_si128(x7, x11, 0x10); + x1 = _mm_clmulepi64_si128(x7, x11, 1); + x0 = _mm_clmulepi64_si128(x7, x11, 0); + x3 = _mm_clmulepi64_si128(x7, x11, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x13 = _mm_xor_si128(x13, x2); + x12 = _mm_xor_si128(x12, x0); + x7 = _mm_bslli_si128(x13, 8); + x13 = _mm_bsrli_si128(x13, 8); + x12 = _mm_xor_si128(x12, x7); + x6 = _mm_xor_si128(x6, x13); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x12, x2, 0x10); + x1 = _mm_shuffle_epi32(x12, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_xor_si128(x6, x1); + rcx = (word32)((word32)rcx + 0x40); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_avx2_calc_aad_agg; + } + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 168)); +L_AES_GCM_decrypt_avx2_calc_aad_after_agg: + rdx = (word32)((word32)r11); + rdx = (word32)((word32)rdx & 0xfffffff0); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx2_calc_aad_partial; + } +L_AES_GCM_decrypt_avx2_calc_aad_16_loop: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r12, rcx)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x6, 0x10); + x1 = _mm_clmulepi64_si128(x5, x6, 1); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + x3 = _mm_clmulepi64_si128(x5, x6, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x6 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x6, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_xor_si128(x6, x1); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx2_calc_aad_16_loop; + } +L_AES_GCM_decrypt_avx2_calc_aad_partial: + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx2_calc_aad_done; + } + x0 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_avx2_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx2_calc_aad_loop; + } + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x6, 0x10); + x1 = _mm_clmulepi64_si128(x5, x6, 1); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + x3 = _mm_clmulepi64_si128(x5, x6, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x7 = _mm_xor_si128(x0, x1); + x6 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x7, 31); + x1 = _mm_srli_epi32(x6, 31); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x6 = _mm_or_si128(x6, x2); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x7, x2, 0x10); + x1 = _mm_shuffle_epi32(x7, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_xor_si128(x6, x1); +L_AES_GCM_decrypt_avx2_calc_aad_done: + /* Calculate counter and H */ + x1 = _mm_srli_epi64(x5, 63); + x0 = _mm_slli_epi64(x5, 1); + x1 = _mm_bslli_si128(x1, 8); + x0 = _mm_or_si128(x0, x1); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x5 = _mm_xor_si128(x5, x0); + rbx = (word32)(0); + zf15 = (word32)r10; + zf16 = 0x80; + r13 = (word32)((word32)r10); + if ((zf15) < (zf16)) { + goto L_AES_GCM_decrypt_avx2_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x15); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + /* H ^ 1 and H ^ 2 */ + x9 = _mm_clmulepi64_si128(x5, x5, 0); + x10 = _mm_clmulepi64_si128(x5, x5, 0x11); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x0 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 and H ^ 4 */ + x11 = _mm_clmulepi64_si128(x0, x5, 0x10); + x10 = _mm_clmulepi64_si128(x0, x5, 1); + x9 = _mm_clmulepi64_si128(x0, x5, 0); + x12 = _mm_clmulepi64_si128(x0, x5, 0x11); + x13 = _mm_clmulepi64_si128(x0, x0, 0); + x14 = _mm_clmulepi64_si128(x0, x0, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x2 = _mm_xor_si128(x13, x14); + x1 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x2); + /* H ^ 5 and H ^ 6 */ + x11 = _mm_clmulepi64_si128(x1, x0, 0x10); + x10 = _mm_clmulepi64_si128(x1, x0, 1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x13 = _mm_clmulepi64_si128(x1, x1, 0); + x14 = _mm_clmulepi64_si128(x1, x1, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x0); + /* H ^ 7 and H ^ 8 */ + x11 = _mm_clmulepi64_si128(x2, x1, 0x10); + x10 = _mm_clmulepi64_si128(x2, x1, 1); + x9 = _mm_clmulepi64_si128(x2, x1, 0); + x12 = _mm_clmulepi64_si128(x2, x1, 0x11); + x13 = _mm_clmulepi64_si128(x2, x2, 0); + x14 = _mm_clmulepi64_si128(x2, x2, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x0); +L_AES_GCM_decrypt_avx2_ghash_128: + /* aesenc_128_ghash */ + rcx = (word64)(rdi + rbx); + rdx = (word64)(r8 + rbx); + /* aesenc_ctr */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_epi64, 0)); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x8 = _mm_shuffle_epi8(x0, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_two, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_three, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_four, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_five, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_six, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_seven, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_eight, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + /* aesenc_xor */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + /* aesenc_pclmul_1 */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x1 = _mm_xor_si128(x1, x6); + x5 = _mm_clmulepi64_si128(x1, x2, 0x10); + x3 = _mm_clmulepi64_si128(x1, x2, 1); + x6 = _mm_clmulepi64_si128(x1, x2, 0); + x7 = _mm_clmulepi64_si128(x1, x2, 0x11); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_2 */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x3); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_l */ + x5 = _mm_xor_si128(x5, x2); + x6 = _mm_xor_si128(x6, x4); + x5 = _mm_xor_si128(x5, x3); + x1 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x8 = _mm_aesenc_si128(x8, x4); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_xor_si128(x7, x5); + x3 = _mm_clmulepi64_si128(x6, x0, 0x10); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x6 = _mm_xor_si128(x6, x3); + x3 = _mm_clmulepi64_si128(x6, x0, 0x10); + x12 = _mm_aesenc_si128(x12, x4); + x13 = _mm_aesenc_si128(x13, x4); + x14 = _mm_aesenc_si128(x14, x4); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x6 = _mm_xor_si128(x6, x3); + x6 = _mm_xor_si128(x6, x7); + x15 = _mm_aesenc_si128(x15, x4); + zf17 = (word32)r9; + zf18 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf19 = (word32)r9; + zf20 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_decrypt_avx2_aesenc_128_ghash_avx_done: + /* aesenc_last */ + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + /* aesenc_128_ghash - end */ + rbx = (word32)((word32)rbx + 0x80); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_avx2_ghash_128; + } + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); +L_AES_GCM_decrypt_avx2_done_128: + if (((word32)rbx) >= ((word32)r10)) { + goto L_AES_GCM_decrypt_avx2_done_dec; + } + r13 = (word32)((word32)r10); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_avx2_last_block_done; + } +L_AES_GCM_decrypt_avx2_last_block_start: + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rbx)); + x10 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x12 = _mm_shuffle_epi8(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x12 = _mm_xor_si128(x12, x6); + /* aesenc_gfmul_sb */ + x2 = _mm_clmulepi64_si128(x12, x5, 1); + x3 = _mm_clmulepi64_si128(x12, x5, 0x10); + x1 = _mm_clmulepi64_si128(x12, x5, 0); + x8 = _mm_clmulepi64_si128(x12, x5, 0x11); + x10 = _mm_xor_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 16))); + x3 = _mm_xor_si128(x3, x2); + x2 = _mm_bslli_si128(x3, 8); + x3 = _mm_bsrli_si128(x3, 8); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 32))); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_clmulepi64_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 80))); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_clmulepi64_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 128))); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 144))); + x8 = _mm_xor_si128(x8, x3); + x2 = _mm_xor_si128(x2, x8); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)((word32)r9) < (sword32)(0xb)) { + goto L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last; + } + x10 = _mm_aesenc_si128(x10, x0); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 176))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)((word32)r9) < (sword32)(0xd)) { + goto L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last; + } + x10 = _mm_aesenc_si128(x10, x0); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_decrypt_avx2_aesenc_gfmul_sb_last: + x10 = _mm_aesenclast_si128(x10, x0); + x6 = _mm_xor_si128(x2, x1); + x10 = _mm_xor_si128(x10, x11); + _mm_storeu_si128((__m128i*)WC_PW(r8, rbx), x10); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_avx2_last_block_start; + } +L_AES_GCM_decrypt_avx2_last_block_done: + rcx = (word32)((word32)r10); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_decrypt_avx2_done_dec; + } + /* aesenc_last15_dec */ + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x4 = _mm_xor_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144))); + zf21 = (word32)r9; + zf22 = 0xb; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x1); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176))); + zf23 = (word32)r9; + zf24 = 0xd; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, x1); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208))); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_aesenc_avx_last: + x4 = _mm_aesenclast_si128(x4, x1); + rcx = (word32)(0); + x0 = _mm_setzero_si128(); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); +L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + WC_S8(rsp, rcx + 16) = (byte)((byte)r13); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(r8, rbx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx2_aesenc_last15_dec_avx_loop; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x4); + /* ghash_gfmul_red */ + x2 = _mm_clmulepi64_si128(x6, x5, 0x10); + x1 = _mm_clmulepi64_si128(x6, x5, 1); + x0 = _mm_clmulepi64_si128(x6, x5, 0); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_clmulepi64_si128(x6, x5, 0x11); + x0 = _mm_clmulepi64_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x6 = _mm_xor_si128(x6, x2); + x6 = _mm_xor_si128(x6, x1); + x6 = _mm_xor_si128(x6, x0); +L_AES_GCM_decrypt_avx2_done_dec: + /* calc_tag */ + r10 = (word64)(r10 << 3); + r11 = (word64)(r11 << 3); + x0 = _mm_cvtsi64_si128((long long)r10); + x1 = _mm_cvtsi64_si128((long long)r11); + x0 = _mm_unpacklo_epi64(x0, x1); + x0 = _mm_xor_si128(x0, x6); + /* ghash_gfmul_red */ + x4 = _mm_clmulepi64_si128(x0, x5, 0x10); + x3 = _mm_clmulepi64_si128(x0, x5, 1); + x2 = _mm_clmulepi64_si128(x0, x5, 0); + x4 = _mm_xor_si128(x4, x3); + x3 = _mm_bslli_si128(x4, 8); + x4 = _mm_bsrli_si128(x4, 8); + x3 = _mm_xor_si128(x3, x2); + x0 = _mm_clmulepi64_si128(x0, x5, 0x11); + x2 = _mm_clmulepi64_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x3 = _mm_shuffle_epi32(x3, 0x4e); + x3 = _mm_xor_si128(x3, x2); + x2 = _mm_clmulepi64_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x3 = _mm_shuffle_epi32(x3, 0x4e); + x0 = _mm_xor_si128(x0, x4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x15); + /* cmp_tag */ + if (((word32)r15) == (0x10)) { + goto L_AES_GCM_decrypt_avx2_cmp_tag_16; + } + rdx = (word64)(0); + rax = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_avx2_cmp_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rdx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(r14, + rdx)) & 0xff); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax | ( + byte)r13) & 0xff); + rdx = (word32)((word32)rdx + 1); + if (((word32)rdx) != ((word32)r15)) { + goto L_AES_GCM_decrypt_avx2_cmp_tag_loop; + } + rax = (rax & ~(word64)0xff) | ((word64)(byte)(((byte)rax) == (0)) & 0xff); + goto L_AES_GCM_decrypt_avx2_cmp_tag_done; +L_AES_GCM_decrypt_avx2_cmp_tag_16: + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 0)); + x0 = _mm_cmpeq_epi8(x0, x1); + rdx = (word32)((word32)_mm_movemask_epi8(x0)); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + rax = (word32)(0); + rax = (rax & ~(word64)0xff) | ((word64)(byte)(((word32)rdx) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_avx2_cmp_tag_done: + WC_S32(rbp, 0) = (word32)((word32)rax); +} + +#ifdef WOLFSSL_AESGCM_STREAM +WC_X64I_TARGET("aes,pclmul,avx2") +WOLFSSL_LOCAL void AES_GCM_init_avx2(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, rsp, rdx, rcx = 0, rbx = 0, + r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)ivec; + r11 = (word64)(word32)ibytes; + r8 = (word64)(size_t)h; + r9 = (word64)(size_t)counter; + rax = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_setzero_si128(); + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0xc)) { + goto L_AES_GCM_init_avx2_iv_12; + } + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf1 = (word32)rsi; + zf2 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_init_avx2_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf3 = (word32)rsi; + zf4 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_init_avx2_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_avx2_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x0); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf5 = (word32)rdx; + zf6 = 0; + rcx = (word64)(0); + if ((zf5) == (zf6)) { + goto L_AES_GCM_init_avx2_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_init_avx2_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_init_avx2_calc_iv_16_loop: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, rcx)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x6 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x6, x2, 0x10); + x1 = _mm_shuffle_epi32(x6, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_avx2_calc_iv_16_loop; + } + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_init_avx2_calc_iv_done; + } +L_AES_GCM_init_avx2_calc_iv_lt16: + x0 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_init_avx2_calc_iv_loop: + r12 = (word32)((word32)WC_L8(r10, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r12); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_avx2_calc_iv_loop; + } + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x6 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x6, x2, 0x10); + x1 = _mm_shuffle_epi32(x6, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); +L_AES_GCM_init_avx2_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x6 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x6, x2, 0x10); + x1 = _mm_shuffle_epi32(x6, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf7 = (word32)rsi; + zf8 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_init_avx2_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x0); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf9 = (word32)rsi; + zf10 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_init_avx2_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x0); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_avx2_calc_iv_2_aesenc_avx_last: + x7 = _mm_aesenclast_si128(x7, x0); + goto L_AES_GCM_init_avx2_iv_done; +L_AES_GCM_init_avx2_iv_12: + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_one, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x4 = _mm_blend_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)), 7); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x7 = _mm_xor_si128(x4, x5); + x5 = _mm_aesenc_si128(x5, x6); + x7 = _mm_aesenc_si128(x7, x6); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + zf11 = (word32)rsi; + zf12 = 0xb; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_init_avx2_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + zf13 = (word32)rsi; + zf14 = 0xd; + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_init_avx2_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x5 = _mm_aesenc_si128(x5, x0); + x7 = _mm_aesenc_si128(x7, x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_avx2_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x0); + x7 = _mm_aesenclast_si128(x7, x0); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); +L_AES_GCM_init_avx2_iv_done: + _mm_storeu_si128((__m128i*)WC_PW(rax, 0), x7); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x4); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_aad_update_avx2(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, x5, + x6 = _mm_setzero_si128(); + + rdi = (word64)(size_t)addt; + rsi = (word64)(word32)abytes; + rdx = (word64)(size_t)tag; + rax = (word64)(size_t)h; + + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + rcx = (word32)(0); +L_AES_GCM_aad_update_avx2_16_loop: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rcx)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x6 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x6, x2, 0x10); + x1 = _mm_shuffle_epi32(x6, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rsi)) { + goto L_AES_GCM_aad_update_avx2_16_loop; + } + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x4); +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_block_avx2(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, rsp; + __m128i x0, x1, x2, x3; + XALIGNED(32) WC_X64I_SLOT stk[20]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 152); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + /* aesenc_block */ + x1 = x3; + x0 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x0 = _mm_aesenc_si128(x0, x2); + x3 = x1; + zf1 = (word32)rsi; + zf2 = 0xb; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_block_avx2_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x0 = _mm_aesenc_si128(x0, x2); + zf3 = (word32)rsi; + zf4 = 0xd; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_block_avx2_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x0 = _mm_aesenc_si128(x0, x2); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_block_avx2_aesenc_block_last: + x0 = _mm_aesenclast_si128(x0, x1); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x3); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_ghash_block_avx2(const unsigned char* addt, + unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, rdx; + __m128i x0, x1, x2, x3, x4, x5, x6; + + rdi = (word64)(size_t)addt; + rsi = (word64)(size_t)tag; + rdx = (word64)(size_t)h; + + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x2 = _mm_clmulepi64_si128(x5, x4, 0x10); + x1 = _mm_clmulepi64_si128(x5, x4, 1); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_bslli_si128(x2, 8); + x2 = _mm_bsrli_si128(x2, 8); + x6 = _mm_xor_si128(x0, x1); + x4 = _mm_xor_si128(x3, x2); + /* ghash_mid */ + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x6, x2, 0x10); + x1 = _mm_shuffle_epi32(x6, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x4, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x4); +} + +WC_X64I_TARGET("aes,pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_update_avx2(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, r13, rcx = 0, + rdx = 0; + __m128i x0, x1, x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5, x6, x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), + x13 = _mm_setzero_si128(), x14 = _mm_setzero_si128(), + x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[20]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 152); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x1 = _mm_srli_epi64(x5, 63); + x0 = _mm_slli_epi64(x5, 1); + x1 = _mm_bslli_si128(x1, 8); + x0 = _mm_or_si128(x0, x1); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x0); + r14 = (word32)(0); + zf1 = (word32)r8; + zf2 = 0x80; + r13 = (word32)((word32)r8); + if ((zf1) < (zf2)) { + goto L_AES_GCM_encrypt_update_avx2_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x4); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + /* H ^ 1 and H ^ 2 */ + x9 = _mm_clmulepi64_si128(x5, x5, 0); + x10 = _mm_clmulepi64_si128(x5, x5, 0x11); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x0 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 and H ^ 4 */ + x11 = _mm_clmulepi64_si128(x0, x5, 0x10); + x10 = _mm_clmulepi64_si128(x0, x5, 1); + x9 = _mm_clmulepi64_si128(x0, x5, 0); + x12 = _mm_clmulepi64_si128(x0, x5, 0x11); + x13 = _mm_clmulepi64_si128(x0, x0, 0); + x14 = _mm_clmulepi64_si128(x0, x0, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x2 = _mm_xor_si128(x13, x14); + x1 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x2); + /* H ^ 5 and H ^ 6 */ + x11 = _mm_clmulepi64_si128(x1, x0, 0x10); + x10 = _mm_clmulepi64_si128(x1, x0, 1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x13 = _mm_clmulepi64_si128(x1, x1, 0); + x14 = _mm_clmulepi64_si128(x1, x1, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x0); + /* H ^ 7 and H ^ 8 */ + x11 = _mm_clmulepi64_si128(x2, x1, 0x10); + x10 = _mm_clmulepi64_si128(x2, x1, 1); + x9 = _mm_clmulepi64_si128(x2, x1, 0); + x12 = _mm_clmulepi64_si128(x2, x1, 0x11); + x13 = _mm_clmulepi64_si128(x2, x2, 0); + x14 = _mm_clmulepi64_si128(x2, x2, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x0); + /* First 128 bytes of input */ + /* aesenc_128 */ + /* aesenc_ctr */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_epi64, 0)); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x8 = _mm_shuffle_epi8(x0, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_two, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_three, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_four, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_five, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_six, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_seven, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_eight, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + /* aesenc_xor */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf3 = (word32)rsi; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_update_avx2_aesenc_128_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf5 = (word32)rsi; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_encrypt_update_avx2_aesenc_128_enc_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_avx2_aesenc_128_enc_done: + /* aesenc_last */ + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 48)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(r10, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(r10, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 112)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + _mm_storeu_si128((__m128i*)WC_PW(r10, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(r10, 80), x13); + _mm_storeu_si128((__m128i*)WC_PW(r10, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(r10, 112), x15); + zf7 = (word32)r13; + zf8 = 0x80; + r14 = (word32)(0x80); + if ((zf7) <= (zf8)) { + goto L_AES_GCM_encrypt_update_avx2_end_128; + } + /* More 128 bytes of input */ +L_AES_GCM_encrypt_update_avx2_ghash_128: + /* aesenc_128_ghash */ + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + /* aesenc_ctr */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_epi64, 0)); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x8 = _mm_shuffle_epi8(x0, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_two, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_three, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_four, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_five, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_six, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_seven, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_eight, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + /* aesenc_xor */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + /* aesenc_pclmul_1 */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -128)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x1 = _mm_xor_si128(x1, x6); + x5 = _mm_clmulepi64_si128(x1, x2, 0x10); + x3 = _mm_clmulepi64_si128(x1, x2, 1); + x6 = _mm_clmulepi64_si128(x1, x2, 0); + x7 = _mm_clmulepi64_si128(x1, x2, 0x11); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_2 */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x3); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, -16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_l */ + x5 = _mm_xor_si128(x5, x2); + x6 = _mm_xor_si128(x6, x4); + x5 = _mm_xor_si128(x5, x3); + x1 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x8 = _mm_aesenc_si128(x8, x4); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_xor_si128(x7, x5); + x3 = _mm_clmulepi64_si128(x6, x0, 0x10); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x6 = _mm_xor_si128(x6, x3); + x3 = _mm_clmulepi64_si128(x6, x0, 0x10); + x12 = _mm_aesenc_si128(x12, x4); + x13 = _mm_aesenc_si128(x13, x4); + x14 = _mm_aesenc_si128(x14, x4); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x6 = _mm_xor_si128(x6, x3); + x6 = _mm_xor_si128(x6, x7); + x15 = _mm_aesenc_si128(x15, x4); + zf9 = (word32)rsi; + zf10 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_encrypt_update_avx2_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf11 = (word32)rsi; + zf12 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_encrypt_update_avx2_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_avx2_aesenc_128_ghash_avx_done: + /* aesenc_last */ + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + /* aesenc_128_ghash - end */ + r14 = (word32)((word32)r14 + 0x80); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx2_ghash_128; + } +L_AES_GCM_encrypt_update_avx2_end_128: + x4 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_mask, 0)); + x8 = _mm_shuffle_epi8(x8, x4); + x9 = _mm_shuffle_epi8(x9, x4); + x10 = _mm_shuffle_epi8(x10, x4); + x11 = _mm_shuffle_epi8(x11, x4); + x12 = _mm_shuffle_epi8(x12, x4); + x13 = _mm_shuffle_epi8(x13, x4); + x14 = _mm_shuffle_epi8(x14, x4); + x15 = _mm_shuffle_epi8(x15, x4); + x8 = _mm_xor_si128(x8, x6); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x5 = _mm_clmulepi64_si128(x7, x15, 0x10); + x1 = _mm_clmulepi64_si128(x7, x15, 1); + x4 = _mm_clmulepi64_si128(x7, x15, 0); + x6 = _mm_clmulepi64_si128(x7, x15, 0x11); + x5 = _mm_xor_si128(x5, x1); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x2 = _mm_clmulepi64_si128(x7, x14, 0x10); + x1 = _mm_clmulepi64_si128(x7, x14, 1); + x0 = _mm_clmulepi64_si128(x7, x14, 0); + x3 = _mm_clmulepi64_si128(x7, x14, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x2 = _mm_clmulepi64_si128(x15, x13, 0x10); + x1 = _mm_clmulepi64_si128(x15, x13, 1); + x0 = _mm_clmulepi64_si128(x15, x13, 0); + x3 = _mm_clmulepi64_si128(x15, x13, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x2 = _mm_clmulepi64_si128(x7, x12, 0x10); + x1 = _mm_clmulepi64_si128(x7, x12, 1); + x0 = _mm_clmulepi64_si128(x7, x12, 0); + x3 = _mm_clmulepi64_si128(x7, x12, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x2 = _mm_clmulepi64_si128(x15, x11, 0x10); + x1 = _mm_clmulepi64_si128(x15, x11, 1); + x0 = _mm_clmulepi64_si128(x15, x11, 0); + x3 = _mm_clmulepi64_si128(x15, x11, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x2 = _mm_clmulepi64_si128(x7, x10, 0x10); + x1 = _mm_clmulepi64_si128(x7, x10, 1); + x0 = _mm_clmulepi64_si128(x7, x10, 0); + x3 = _mm_clmulepi64_si128(x7, x10, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x2 = _mm_clmulepi64_si128(x15, x9, 0x10); + x1 = _mm_clmulepi64_si128(x15, x9, 1); + x0 = _mm_clmulepi64_si128(x15, x9, 0); + x3 = _mm_clmulepi64_si128(x15, x9, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x2 = _mm_clmulepi64_si128(x7, x8, 0x10); + x1 = _mm_clmulepi64_si128(x7, x8, 1); + x0 = _mm_clmulepi64_si128(x7, x8, 0); + x3 = _mm_clmulepi64_si128(x7, x8, 0x11); + x2 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x6, x3); + x5 = _mm_xor_si128(x5, x2); + x4 = _mm_xor_si128(x4, x0); + x7 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x4 = _mm_xor_si128(x4, x7); + x6 = _mm_xor_si128(x6, x5); + /* ghash_red */ + x2 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x0 = _mm_clmulepi64_si128(x4, x2, 0x10); + x1 = _mm_shuffle_epi32(x4, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x0 = _mm_clmulepi64_si128(x1, x2, 0x10); + x1 = _mm_shuffle_epi32(x1, 0x4e); + x1 = _mm_xor_si128(x1, x0); + x6 = _mm_xor_si128(x6, x1); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); +L_AES_GCM_encrypt_update_avx2_done_128: + if (((word32)r14) == ((word32)r8)) { + goto L_AES_GCM_encrypt_update_avx2_done_enc; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx2_last_block_done; + } + /* aesenc_block */ + x1 = x4; + x0 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x0 = _mm_aesenc_si128(x0, x2); + x4 = x1; + zf13 = (word32)rsi; + zf14 = 0xb; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_update_avx2_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x0 = _mm_aesenc_si128(x0, x2); + zf15 = (word32)rsi; + zf16 = 0xd; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_encrypt_update_avx2_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x0 = _mm_aesenc_si128(x0, x2); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_avx2_aesenc_block_last: + x0 = _mm_aesenclast_si128(x0, x1); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, r14)); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), x0); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x0); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx2_last_block_ghash; + } +L_AES_GCM_encrypt_update_avx2_last_block_start: + x12 = _mm_loadu_si128((const __m128i*)WC_PR(r11, r14)); + x11 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + /* aesenc_gfmul_sb */ + x2 = _mm_clmulepi64_si128(x6, x5, 1); + x3 = _mm_clmulepi64_si128(x6, x5, 0x10); + x1 = _mm_clmulepi64_si128(x6, x5, 0); + x8 = _mm_clmulepi64_si128(x6, x5, 0x11); + x11 = _mm_xor_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x3 = _mm_xor_si128(x3, x2); + x2 = _mm_bslli_si128(x3, 8); + x3 = _mm_bsrli_si128(x3, 8); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_clmulepi64_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_clmulepi64_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x8 = _mm_xor_si128(x8, x3); + x2 = _mm_xor_si128(x2, x8); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)((word32)rsi) < (sword32)(0xb)) { + goto L_AES_GCM_encrypt_update_avx2_aesenc_gfmul_sb_last; + } + x11 = _mm_aesenc_si128(x11, x0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)((word32)rsi) < (sword32)(0xd)) { + goto L_AES_GCM_encrypt_update_avx2_aesenc_gfmul_sb_last; + } + x11 = _mm_aesenc_si128(x11, x0); + x11 = _mm_aesenc_si128(x11, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_update_avx2_aesenc_gfmul_sb_last: + x11 = _mm_aesenclast_si128(x11, x0); + x6 = _mm_xor_si128(x2, x1); + x11 = _mm_xor_si128(x11, x12); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), x11); + x11 = _mm_shuffle_epi8(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x11); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx2_last_block_start; + } +L_AES_GCM_encrypt_update_avx2_last_block_ghash: + /* ghash_gfmul_red */ + x10 = _mm_clmulepi64_si128(x6, x5, 0x10); + x9 = _mm_clmulepi64_si128(x6, x5, 1); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + x10 = _mm_xor_si128(x10, x9); + x9 = _mm_bslli_si128(x10, 8); + x10 = _mm_bsrli_si128(x10, 8); + x9 = _mm_xor_si128(x9, x8); + x6 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_clmulepi64_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x8 = _mm_clmulepi64_si128(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x6 = _mm_xor_si128(x6, x10); + x6 = _mm_xor_si128(x6, x9); + x6 = _mm_xor_si128(x6, x8); +L_AES_GCM_encrypt_update_avx2_last_block_done: +L_AES_GCM_encrypt_update_avx2_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x6); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x4); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_final_avx2(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, rsp, r10 = 0, r11 = 0; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rdx = (word64)(word32)tbytes; + rcx = (word64)(word32)nbytes; + r8 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + rax = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x1 = _mm_srli_epi64(x5, 63); + x0 = _mm_slli_epi64(x5, 1); + x1 = _mm_bslli_si128(x1, 8); + x0 = _mm_or_si128(x0, x1); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x0); + /* calc_tag */ + rcx = (word64)(rcx << 3); + r8 = (word64)(r8 << 3); + x0 = _mm_cvtsi64_si128((long long)rcx); + x1 = _mm_cvtsi64_si128((long long)r8); + x0 = _mm_unpacklo_epi64(x0, x1); + x0 = _mm_xor_si128(x0, x4); + /* ghash_gfmul_red */ + x7 = _mm_clmulepi64_si128(x0, x5, 0x10); + x3 = _mm_clmulepi64_si128(x0, x5, 1); + x2 = _mm_clmulepi64_si128(x0, x5, 0); + x7 = _mm_xor_si128(x7, x3); + x3 = _mm_bslli_si128(x7, 8); + x7 = _mm_bsrli_si128(x7, 8); + x3 = _mm_xor_si128(x3, x2); + x0 = _mm_clmulepi64_si128(x0, x5, 0x11); + x2 = _mm_clmulepi64_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x3 = _mm_shuffle_epi32(x3, 0x4e); + x3 = _mm_xor_si128(x3, x2); + x2 = _mm_clmulepi64_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x3 = _mm_shuffle_epi32(x3, 0x4e); + x0 = _mm_xor_si128(x0, x7); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x6); + /* store_tag */ + if (((word32)rdx) == (0x10)) { + goto L_AES_GCM_encrypt_final_avx2_store_tag_16; + } + r10 = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_final_avx2_store_tag_loop: + r11 = (word32)((word32)WC_L8(rsp, r10)); + WC_S8(rsi, r10) = (byte)((byte)r11); + r10 = (word32)((word32)r10 + 1); + if (((word32)r10) != ((word32)rdx)) { + goto L_AES_GCM_encrypt_final_avx2_store_tag_loop; + } + goto L_AES_GCM_encrypt_final_avx2_store_tag_done; +L_AES_GCM_encrypt_final_avx2_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); +L_AES_GCM_encrypt_final_avx2_store_tag_done: + ; +} + +WC_X64I_TARGET("aes,pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_decrypt_update_avx2(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, r13, rcx = 0, + rdx = 0; + __m128i x0, x1, x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5, x6, x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), + x13 = _mm_setzero_si128(), x14 = _mm_setzero_si128(), + x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[24]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 168); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + /* Calculate H */ + x1 = _mm_srli_epi64(x5, 63); + x0 = _mm_slli_epi64(x5, 1); + x1 = _mm_bslli_si128(x1, 8); + x0 = _mm_or_si128(x0, x1); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x0); + r14 = (word32)(0); + zf1 = (word32)r8; + zf2 = 0x80; + r13 = (word32)((word32)r8); + if ((zf1) < (zf2)) { + goto L_AES_GCM_decrypt_update_avx2_done_128; + } + r13 = (word32)((word32)r13 & 0xffffff80); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x15); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + /* H ^ 1 and H ^ 2 */ + x9 = _mm_clmulepi64_si128(x5, x5, 0); + x10 = _mm_clmulepi64_si128(x5, x5, 0x11); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x8 = _mm_clmulepi64_si128(x9, x3, 0x10); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x9 = _mm_xor_si128(x9, x8); + x0 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 and H ^ 4 */ + x11 = _mm_clmulepi64_si128(x0, x5, 0x10); + x10 = _mm_clmulepi64_si128(x0, x5, 1); + x9 = _mm_clmulepi64_si128(x0, x5, 0); + x12 = _mm_clmulepi64_si128(x0, x5, 0x11); + x13 = _mm_clmulepi64_si128(x0, x0, 0); + x14 = _mm_clmulepi64_si128(x0, x0, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x2 = _mm_xor_si128(x13, x14); + x1 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x2); + /* H ^ 5 and H ^ 6 */ + x11 = _mm_clmulepi64_si128(x1, x0, 0x10); + x10 = _mm_clmulepi64_si128(x1, x0, 1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x13 = _mm_clmulepi64_si128(x1, x1, 0); + x14 = _mm_clmulepi64_si128(x1, x1, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x0); + /* H ^ 7 and H ^ 8 */ + x11 = _mm_clmulepi64_si128(x2, x1, 0x10); + x10 = _mm_clmulepi64_si128(x2, x1, 1); + x9 = _mm_clmulepi64_si128(x2, x1, 0); + x12 = _mm_clmulepi64_si128(x2, x1, 0x11); + x13 = _mm_clmulepi64_si128(x2, x2, 0); + x14 = _mm_clmulepi64_si128(x2, x2, 0x11); + x11 = _mm_xor_si128(x11, x10); + x10 = _mm_bslli_si128(x11, 8); + x11 = _mm_bsrli_si128(x11, 8); + x10 = _mm_xor_si128(x10, x9); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x10 = _mm_xor_si128(x10, x9); + x13 = _mm_xor_si128(x13, x8); + x9 = _mm_clmulepi64_si128(x10, x3, 0x10); + x8 = _mm_clmulepi64_si128(x13, x3, 0x10); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x13 = _mm_shuffle_epi32(x13, 0x4e); + x12 = _mm_xor_si128(x12, x11); + x13 = _mm_xor_si128(x13, x8); + x10 = _mm_xor_si128(x10, x12); + x0 = _mm_xor_si128(x13, x14); + x7 = _mm_xor_si128(x10, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x0); +L_AES_GCM_decrypt_update_avx2_ghash_128: + /* aesenc_128_ghash */ + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + /* aesenc_ctr */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_bswap_epi64, 0)); + x9 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x8 = _mm_shuffle_epi8(x0, x1); + x10 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_two, 0))); + x9 = _mm_shuffle_epi8(x9, x1); + x11 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_three, 0))); + x10 = _mm_shuffle_epi8(x10, x1); + x12 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_four, 0))); + x11 = _mm_shuffle_epi8(x11, x1); + x13 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_five, 0))); + x12 = _mm_shuffle_epi8(x12, x1); + x14 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_six, 0))); + x13 = _mm_shuffle_epi8(x13, x1); + x15 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_seven, 0))); + x14 = _mm_shuffle_epi8(x14, x1); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_eight, 0))); + x15 = _mm_shuffle_epi8(x15, x1); + /* aesenc_xor */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x0); + x8 = _mm_xor_si128(x8, x7); + x9 = _mm_xor_si128(x9, x7); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_xor_si128(x11, x7); + x12 = _mm_xor_si128(x12, x7); + x13 = _mm_xor_si128(x13, x7); + x14 = _mm_xor_si128(x14, x7); + x15 = _mm_xor_si128(x15, x7); + /* aesenc_pclmul_1 */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x1 = _mm_xor_si128(x1, x6); + x5 = _mm_clmulepi64_si128(x1, x2, 0x10); + x3 = _mm_clmulepi64_si128(x1, x2, 1); + x6 = _mm_clmulepi64_si128(x1, x2, 0); + x7 = _mm_clmulepi64_si128(x1, x2, 0x11); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_2 */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x3); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 32)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_n */ + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x1 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x5 = _mm_xor_si128(x5, x2); + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x5 = _mm_xor_si128(x5, x3); + x3 = _mm_clmulepi64_si128(x1, x0, 1); + x6 = _mm_xor_si128(x6, x4); + x4 = _mm_clmulepi64_si128(x1, x0, 0); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x7 = _mm_xor_si128(x7, x1); + x8 = _mm_aesenc_si128(x8, x0); + x9 = _mm_aesenc_si128(x9, x0); + x10 = _mm_aesenc_si128(x10, x0); + x11 = _mm_aesenc_si128(x11, x0); + x12 = _mm_aesenc_si128(x12, x0); + x13 = _mm_aesenc_si128(x13, x0); + x14 = _mm_aesenc_si128(x14, x0); + x15 = _mm_aesenc_si128(x15, x0); + /* aesenc_pclmul_l */ + x5 = _mm_xor_si128(x5, x2); + x6 = _mm_xor_si128(x6, x4); + x5 = _mm_xor_si128(x5, x3); + x1 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_aes_gcm_mod2_128, 0)); + x8 = _mm_aesenc_si128(x8, x4); + x6 = _mm_xor_si128(x6, x1); + x7 = _mm_xor_si128(x7, x5); + x3 = _mm_clmulepi64_si128(x6, x0, 0x10); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x6 = _mm_xor_si128(x6, x3); + x3 = _mm_clmulepi64_si128(x6, x0, 0x10); + x12 = _mm_aesenc_si128(x12, x4); + x13 = _mm_aesenc_si128(x13, x4); + x14 = _mm_aesenc_si128(x14, x4); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x6 = _mm_xor_si128(x6, x3); + x6 = _mm_xor_si128(x6, x7); + x15 = _mm_aesenc_si128(x15, x4); + zf3 = (word32)rsi; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_update_avx2_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + zf5 = (word32)rsi; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_decrypt_update_avx2_aesenc_128_ghash_avx_done; + } + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x8 = _mm_aesenc_si128(x8, x7); + x9 = _mm_aesenc_si128(x9, x7); + x10 = _mm_aesenc_si128(x10, x7); + x11 = _mm_aesenc_si128(x11, x7); + x12 = _mm_aesenc_si128(x12, x7); + x13 = _mm_aesenc_si128(x13, x7); + x14 = _mm_aesenc_si128(x14, x7); + x15 = _mm_aesenc_si128(x15, x7); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_decrypt_update_avx2_aesenc_128_ghash_avx_done: + /* aesenc_last */ + x8 = _mm_aesenclast_si128(x8, x7); + x9 = _mm_aesenclast_si128(x9, x7); + x10 = _mm_aesenclast_si128(x10, x7); + x11 = _mm_aesenclast_si128(x11, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x12 = _mm_aesenclast_si128(x12, x7); + x13 = _mm_aesenclast_si128(x13, x7); + x14 = _mm_aesenclast_si128(x14, x7); + x15 = _mm_aesenclast_si128(x15, x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x13); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x15); + /* aesenc_128_ghash - end */ + r14 = (word32)((word32)r14 + 0x80); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx2_ghash_128; + } + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); +L_AES_GCM_decrypt_update_avx2_done_128: + if (((word32)r14) >= ((word32)r8)) { + goto L_AES_GCM_decrypt_update_avx2_done_dec; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx2_last_block_done; + } +L_AES_GCM_decrypt_update_avx2_last_block_start: + x11 = _mm_loadu_si128((const __m128i*)WC_PR(r11, r14)); + x10 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_epi64, 0))); + x12 = _mm_shuffle_epi8(x11, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_one, 0))); + x12 = _mm_xor_si128(x12, x6); + /* aesenc_gfmul_sb */ + x2 = _mm_clmulepi64_si128(x12, x5, 1); + x3 = _mm_clmulepi64_si128(x12, x5, 0x10); + x1 = _mm_clmulepi64_si128(x12, x5, 0); + x8 = _mm_clmulepi64_si128(x12, x5, 0x11); + x10 = _mm_xor_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + x3 = _mm_xor_si128(x3, x2); + x2 = _mm_bslli_si128(x3, 8); + x3 = _mm_bsrli_si128(x3, 8); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_clmulepi64_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x2 = _mm_xor_si128(x2, x1); + x1 = _mm_clmulepi64_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + x8 = _mm_xor_si128(x8, x3); + x2 = _mm_xor_si128(x2, x8); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)((word32)rsi) < (sword32)(0xb)) { + goto L_AES_GCM_decrypt_update_avx2_aesenc_gfmul_sb_last; + } + x10 = _mm_aesenc_si128(x10, x0); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)((word32)rsi) < (sword32)(0xd)) { + goto L_AES_GCM_decrypt_update_avx2_aesenc_gfmul_sb_last; + } + x10 = _mm_aesenc_si128(x10, x0); + x10 = _mm_aesenc_si128(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_decrypt_update_avx2_aesenc_gfmul_sb_last: + x10 = _mm_aesenclast_si128(x10, x0); + x6 = _mm_xor_si128(x2, x1); + x10 = _mm_xor_si128(x10, x11); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), x10); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx2_last_block_start; + } +L_AES_GCM_decrypt_update_avx2_last_block_done: +L_AES_GCM_decrypt_update_avx2_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x6); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x4); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_decrypt_final_avx2(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, rsp, r11 = 0, r12 = 0; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rdx = (word64)(word32)tbytes; + rcx = (word64)(word32)nbytes; + r8 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + rax = (word64)(size_t)initCtr; + r10 = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x1 = _mm_srli_epi64(x5, 63); + x0 = _mm_slli_epi64(x5, 1); + x1 = _mm_bslli_si128(x1, 8); + x0 = _mm_or_si128(x0, x1); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x0); + /* calc_tag */ + rcx = (word64)(rcx << 3); + r8 = (word64)(r8 << 3); + x0 = _mm_cvtsi64_si128((long long)rcx); + x1 = _mm_cvtsi64_si128((long long)r8); + x0 = _mm_unpacklo_epi64(x0, x1); + x0 = _mm_xor_si128(x0, x4); + /* ghash_gfmul_red */ + x7 = _mm_clmulepi64_si128(x0, x5, 0x10); + x3 = _mm_clmulepi64_si128(x0, x5, 1); + x2 = _mm_clmulepi64_si128(x0, x5, 0); + x7 = _mm_xor_si128(x7, x3); + x3 = _mm_bslli_si128(x7, 8); + x7 = _mm_bsrli_si128(x7, 8); + x3 = _mm_xor_si128(x3, x2); + x0 = _mm_clmulepi64_si128(x0, x5, 0x11); + x2 = _mm_clmulepi64_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x3 = _mm_shuffle_epi32(x3, 0x4e); + x3 = _mm_xor_si128(x3, x2); + x2 = _mm_clmulepi64_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_mod2_128, 0)), 0x10); + x3 = _mm_shuffle_epi32(x3, 0x4e); + x0 = _mm_xor_si128(x0, x7); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx2_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x0, x6); + /* cmp_tag */ + if (((word32)rdx) == (0x10)) { + goto L_AES_GCM_decrypt_final_avx2_cmp_tag_16; + } + r11 = (word64)(0); + r9 = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_final_avx2_cmp_tag_loop: + r12 = (word32)((word32)WC_L8(rsp, r11)); + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)((byte)r12 ^ WC_L8(rsi, + r11)) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)r9 | (byte)r12) & 0xff); + r11 = (word32)((word32)r11 + 1); + if (((word32)r11) != ((word32)rdx)) { + goto L_AES_GCM_decrypt_final_avx2_cmp_tag_loop; + } + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(((byte)r9) == (0)) & 0xff); + goto L_AES_GCM_decrypt_final_avx2_cmp_tag_done; +L_AES_GCM_decrypt_final_avx2_cmp_tag_16: + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_cmpeq_epi8(x0, x1); + r11 = (word32)((word32)_mm_movemask_epi8(x0)); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + r9 = (word32)(0); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(((word32)r11) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_final_avx2_cmp_tag_done: + WC_S32(r10, 0) = (word32)((word32)r9); +} + +#endif /* WOLFSSL_AESGCM_STREAM */ +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_VAES +XALIGNED(32) static const word64 L_vaes_aes_gcm_inc_y0[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, +}; + +XALIGNED(32) static const word64 L_vaes_aes_gcm_inc_y1[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000003ULL, +}; + +XALIGNED(32) static const word64 L_vaes_aes_gcm_inc_y2[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, + 0x0000000000000000ULL, 0x0000000000000005ULL, +}; + +XALIGNED(32) static const word64 L_vaes_aes_gcm_inc_y3[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000007ULL, +}; + +XALIGNED(16) static const word64 L_GCM_generate_m0_vaes_rev8[] + WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_GCM_generate_m0_vaes_mod2_128[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0xe100000000000000ULL, +}; + +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL void GCM_generate_m0_vaes(const unsigned char* h, + unsigned char* m) +{ + word64 rdi, rsi; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10; + + rdi = (word64)(size_t)h; + rsi = (word64)(size_t)m; + + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_GCM_generate_m0_vaes_rev8, 0)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(L_GCM_generate_m0_vaes_mod2_128, + 0)); + x8 = _mm_setzero_si128(); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x8); + x8 = x0; + x0 = _mm_shuffle_epi8(x0, x9); + x5 = _mm_slli_epi64(x0, 63); + x4 = _mm_srli_epi64(x0, 1); + x1 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x1 = _mm_shuffle_epi32(x1, 0xff); + x4 = _mm_or_si128(x4, x5); + x1 = _mm_srai_epi32(x1, 31); + x1 = _mm_and_si128(x1, x10); + x1 = _mm_xor_si128(x1, x4); + x5 = _mm_slli_epi64(x1, 63); + x4 = _mm_srli_epi64(x1, 1); + x2 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x2 = _mm_shuffle_epi32(x2, 0xff); + x4 = _mm_or_si128(x4, x5); + x2 = _mm_srai_epi32(x2, 31); + x2 = _mm_and_si128(x2, x10); + x2 = _mm_xor_si128(x2, x4); + x5 = _mm_slli_epi64(x2, 63); + x4 = _mm_srli_epi64(x2, 1); + x3 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x3 = _mm_shuffle_epi32(x3, 0xff); + x4 = _mm_or_si128(x4, x5); + x3 = _mm_srai_epi32(x3, 31); + x3 = _mm_and_si128(x3, x10); + x3 = _mm_xor_si128(x3, x4); + x3 = _mm_shuffle_epi8(x3, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x0 = _mm_shuffle_epi8(x0, x9); + x8 = _mm_xor_si128(x3, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x3); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x1); + x4 = _mm_xor_si128(x3, x1); + x5 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x8, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 128), x0); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x3, x0); + x6 = _mm_xor_si128(x2, x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 144), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 160), x6); + x6 = _mm_xor_si128(x3, x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 176), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 192), x1); + x4 = _mm_xor_si128(x3, x1); + x5 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x8, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 208), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 224), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 240), x6); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 256), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 272), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 288), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 304), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 320), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 336), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 352), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 368), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 384), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 400), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 416), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 432), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 240)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 448), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 464), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 480), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 496), x3); +} + +XALIGNED(16) static const word64 L_vaes_aes_gcm_one[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000001ULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_two[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_three[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000003ULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_four[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_five[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000005ULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_six[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000006ULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_seven[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000007ULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_eight[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000008ULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_bswap_epi64[] WC_X64I_UNUSED = { + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_bswap_mask[] WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_vaes_aes_gcm_mod2_128[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0xc200000000000000ULL, +}; + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_GCM_encrypt_vaes(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr) +{ + word64 rdi, rsi, r12, rax, r8, r9, r11, rbx, r14, r15, r10, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x15; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5, y6 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(), y8 = _mm256_setzero_si256(), + y9 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(), y14 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[76]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + word32 zf35; + word32 zf36; + word32 zf37; + word32 zf38; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r8 = (word64)(size_t)tag; + r9 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r14 = (word64)(word64)tbytes; + r15 = (word64)(size_t)key; + r10 = (word64)(word64)nr; + + rsp = (word64)(size_t)stk + 608; + rsp = (word64)(rsp - 592); + y5 = _mm256_zextsi128_si256(_mm_setzero_si128()); + x15 = _mm_setzero_si128(); + rdx = (word32)((word32)rbx); + if (((word32)rdx) != (0xc)) { + goto L_AES_GCM_encrypt_vaes_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + y5 = _mm256_zextsi128_si256(_mm_loadl_epi64((const __m128i*)WC_PR(rax, 0))); + y5 = _mm256_zextsi128_si256(_mm_insert_epi32(_mm256_castsi256_si128(y5), ( + int)WC_L32(rax, 8), 2)); + y5 = _mm256_zextsi128_si256(_mm_insert_epi32(_mm256_castsi256_si128(y5), ( + int)(word32)rcx, 3)); + /* H = Encrypt X(=0) and T = Encrypt counter */ + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y6))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + zf1 = (word32)r10; + zf2 = 0xb; + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_12_last; + } + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + zf3 = (word32)r10; + zf4 = 0xd; + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_12_last; + } + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_vaes_calc_iv_12_last: + y6 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 528), _mm256_castsi256_si128(y1)); + goto L_AES_GCM_encrypt_vaes_iv_done; +L_AES_GCM_encrypt_vaes_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)))); + zf5 = (word32)r10; + zf6 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_1_aesenc_avx_last; + } + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)))); + zf7 = (word32)r10; + zf8 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_1_aesenc_avx_last; + } + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_vaes_calc_iv_1_aesenc_avx_last: + y6 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + /* Calc counter */ + /* Initialization vector */ + zf9 = (word32)rdx; + zf10 = 0; + rcx = (word64)(0); + if ((zf9) == (zf10)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_encrypt_vaes_calc_iv_16_loop: + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, + rcx))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y7))); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y5), + 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y5))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y5 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y3)); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y5), 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + y5 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y5), 1)); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_done; + } +L_AES_GCM_encrypt_vaes_calc_iv_lt16: + rsp = (word64)(rsp - 16); + y7 = _mm256_zextsi128_si256(_mm_setzero_si128()); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y7)); +L_AES_GCM_encrypt_vaes_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_loop; + } + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + rsp = (word64)(rsp + 16); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y7))); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y5), + 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y5))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y5 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y3)); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y5), 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + y5 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y5), 1)); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); +L_AES_GCM_encrypt_vaes_calc_iv_done: + /* T = Encrypt counter */ + y0 = _mm256_zextsi128_si256(_mm_setzero_si128()); + rdx = (word32)((word32)rdx << 3); + y0 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rdx)); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y0))); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y5), + 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y5))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y5 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y3)); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y5), 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + y5 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y5), 1)); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + y5 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + /* Encrypt counter */ + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y5))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)))); + zf11 = (word32)r10; + zf12 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_2_aesenc_avx_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)))); + zf13 = (word32)r10; + zf14 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_vaes_calc_iv_2_aesenc_avx_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_vaes_calc_iv_2_aesenc_avx_last: + y7 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 528), _mm256_castsi256_si128(y7)); +L_AES_GCM_encrypt_vaes_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x80)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_ser; + } + _mm_storeu_si128((__m128i*)WC_PW(rsp, 560), _mm256_castsi256_si128(y6)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 576), _mm256_castsi256_si128(y5)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi64(_mm256_castsi256_si128(y6), 63)); + y7 = _mm256_zextsi128_si256(_mm_slli_epi64(_mm256_castsi256_si128(y6), 1)); + y8 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y8), 8)); + y7 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0xff)); + y6 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y6), 31)); + y6 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_mod2_128, 0)))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y2 = _mm256_zextsi128_si256(x15); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y6)); + /* H ^ 2 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y6), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y6), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm256_castsi256_si128(y1)); + /* H ^ 4 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y3 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm256_castsi256_si128(y3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), _mm256_castsi256_si128(y4)); + /* H ^ 6 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y1), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), _mm256_castsi256_si128(y4)); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), _mm256_castsi256_si128(y4)); + /* H ^ 8 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y3), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), _mm256_castsi256_si128(y4)); + if (((word32)rdx) < (0x100)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_no_ext; + } + /* H ^ 9 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 48))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), _mm256_castsi256_si128(y4)); + /* H ^ 10 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), _mm256_castsi256_si128(y4)); + /* H ^ 11 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), _mm256_castsi256_si128(y4)); + /* H ^ 12 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), _mm256_castsi256_si128(y4)); + /* H ^ 13 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), _mm256_castsi256_si128(y4)); + /* H ^ 14 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), _mm256_castsi256_si128(y4)); + /* H ^ 15 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), _mm256_castsi256_si128(y4)); + /* H ^ 16 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), _mm256_castsi256_si128(y4)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 224)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 192)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 160)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 128)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y10); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y10); +L_AES_GCM_encrypt_vaes_calc_aad_no_ext: + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xffffff00); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_a256; + } +L_AES_GCM_encrypt_vaes_calc_aad_l256: + /* 256 bytes of AAD */ + rbx = (word64)(r12 + rcx); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 288)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + rcx = (word32)((word32)rcx + 0x100); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_l256; + } +L_AES_GCM_encrypt_vaes_calc_aad_a256: + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_a128; + } + /* 128 bytes of AAD */ + rbx = (word64)(r12 + rcx); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + rcx = (word32)((word32)rcx + 0x80); +L_AES_GCM_encrypt_vaes_calc_aad_a128: + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_arem; + } + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); +L_AES_GCM_encrypt_vaes_calc_aad_rem: + /* 16 bytes of AAD */ + rbx = (word64)(r12 + rcx); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rbx, 0))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + x15 = _mm256_castsi256_si128(y10); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_rem; + } +L_AES_GCM_encrypt_vaes_calc_aad_arem: + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 560))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 576))); + goto L_AES_GCM_encrypt_vaes_calc_aad_partial; +L_AES_GCM_encrypt_vaes_calc_aad_ser: + rdx = (word32)((word32)r11); + rdx = (word32)((word32)rdx & 0xfffffff0); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_partial; + } +L_AES_GCM_encrypt_vaes_calc_aad_16_loop: + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, + rcx))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(x15, 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + x15, 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + x15, 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), x15)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + x15 = _mm256_castsi256_si128(y3); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(x15, 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + x15 = _mm_slli_epi32(x15, 1); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + x15 = _mm_or_si128(x15, _mm256_castsi256_si128(y2)); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + x15 = _mm_or_si128(x15, _mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y2)); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_16_loop; + } +L_AES_GCM_encrypt_vaes_calc_aad_partial: + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_done; + } + rsp = (word64)(rsp - 16); + y7 = _mm256_zextsi128_si256(_mm_setzero_si128()); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y7)); +L_AES_GCM_encrypt_vaes_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_vaes_calc_aad_loop; + } + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + rsp = (word64)(rsp + 16); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(x15, 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + x15, 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + x15, 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), x15)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + x15 = _mm256_castsi256_si128(y3); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(x15, 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + x15 = _mm_slli_epi32(x15, 1); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + x15 = _mm_or_si128(x15, _mm256_castsi256_si128(y2)); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + x15 = _mm_or_si128(x15, _mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y2)); +L_AES_GCM_encrypt_vaes_calc_aad_done: + /* Calculate counter and H */ + y8 = _mm256_zextsi128_si256(_mm_srli_epi64(_mm256_castsi256_si128(y6), 63)); + y7 = _mm256_zextsi128_si256(_mm_slli_epi64(_mm256_castsi256_si128(y6), 1)); + y8 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y8), 8)); + y7 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0xff)); + y6 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y6), 31)); + y5 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y6 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_mod2_128, 0)))); + y5 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_one, 0)))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y5)); + rbx = (word32)(0); + if (((word32)r9) < (0x80)) { + goto L_AES_GCM_encrypt_vaes_done_128; + } + y2 = _mm256_zextsi128_si256(x15); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y6)); + /* H ^ 2 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y6), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y6), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm256_castsi256_si128(y1)); + /* H ^ 4 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y3 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm256_castsi256_si128(y3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), _mm256_castsi256_si128(y4)); + /* H ^ 6 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y1), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), _mm256_castsi256_si128(y4)); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), _mm256_castsi256_si128(y4)); + /* H ^ 8 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y3), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), _mm256_castsi256_si128(y4)); + if (((word32)r9) < (0x100)) { + goto L_AES_GCM_encrypt_vaes_no_ext; + } + /* H ^ 9 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 48))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), _mm256_castsi256_si128(y4)); + /* H ^ 10 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), _mm256_castsi256_si128(y4)); + /* H ^ 11 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), _mm256_castsi256_si128(y4)); + /* H ^ 12 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), _mm256_castsi256_si128(y4)); + /* H ^ 13 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), _mm256_castsi256_si128(y4)); + /* H ^ 14 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), _mm256_castsi256_si128(y4)); + /* H ^ 15 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), _mm256_castsi256_si128(y4)); + /* H ^ 16 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), _mm256_castsi256_si128(y4)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 224)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 192)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 160)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 128)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y10); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y10); +L_AES_GCM_encrypt_vaes_no_ext: + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + if (((word32)r9) < (0x100)) { + goto L_AES_GCM_encrypt_vaes_after_256; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xffffff00); +L_AES_GCM_encrypt_vaes_loop_256: + /* 256 bytes of input */ + rcx = (word64)(rsi + rbx); + WC_S64(rsp, 544) = (word64)(rcx); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf15 = (word32)r10; + zf16 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_encrypt_vaes_p1_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf17 = (word32)r10; + zf18 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_encrypt_vaes_p1_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_vaes_p1_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + rbx = (word32)((word32)rbx + 0x80); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf19 = (word32)r10; + zf20 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_encrypt_vaes_p2_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf21 = (word32)r10; + zf22 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_encrypt_vaes_p2_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_vaes_p2_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + rbx = (word32)((word32)rbx + 0x80); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + rcx = (word64)(WC_L64(rsp, 544)); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 288)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 128)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 160)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 192)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 224)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_loop_256; + } +L_AES_GCM_encrypt_vaes_after_256: + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_after_128; + } + /* 128 bytes of input */ + rcx = (word64)(rsi + rbx); + WC_S64(rsp, 544) = (word64)(rcx); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf23 = (word32)r10; + zf24 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_encrypt_vaes_8_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf25 = (word32)r10; + zf26 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_encrypt_vaes_8_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_vaes_8_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + rbx = (word32)((word32)rbx + 0x80); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + rcx = (word64)(WC_L64(rsp, 544)); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); +L_AES_GCM_encrypt_vaes_after_128: + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); +L_AES_GCM_encrypt_vaes_done_128: + rdx = (word32)((word32)r9); + if (((word32)rbx) >= ((word32)rdx)) { + goto L_AES_GCM_encrypt_vaes_done_enc; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_last_block_done; + } + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y8 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y8)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)))); + zf27 = (word32)r10; + zf28 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_encrypt_vaes_aesenc_block_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)))); + zf29 = (word32)r10; + zf30 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_encrypt_vaes_aesenc_block_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_vaes_aesenc_block_last: + y7 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + rbx))); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rsi, rbx), _mm256_castsi256_si128(y7)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_last_block_ghash; + } +L_AES_GCM_encrypt_vaes_last_block_start: + y12 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + rbx))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y8 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y8)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)))); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 8)); + y9 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), 8)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y11))); + y3 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y9))); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)))); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y2), + 0x4e)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)))); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y9), + 0x4e)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y9), _mm256_castsi256_si128(y3)); + zf31 = (word32)r10; + zf32 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_GCM_encrypt_vaes_aesenc_gfmul_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)))); + zf33 = (word32)r10; + zf34 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_GCM_encrypt_vaes_aesenc_gfmul_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_vaes_aesenc_gfmul_last: + y7 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y12)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y0))); + _mm_storeu_si128((__m128i*)WC_PW(rsi, rbx), _mm256_castsi256_si128(y7)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + rbx = (word32)((word32)rbx + 0x10); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_vaes_last_block_start; + } +L_AES_GCM_encrypt_vaes_last_block_ghash: + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + x15 = _mm256_castsi256_si128(y10); +L_AES_GCM_encrypt_vaes_last_block_done: + rcx = (word32)((word32)r9); + rdx = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_done; + } + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y5 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)))); + zf35 = (word32)r10; + zf36 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf35) < (sword32)(zf36)) { + goto L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_aesenc_avx_last; + } + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y8))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)))); + zf37 = (word32)r10; + zf38 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf37) < (sword32)(zf38)) { + goto L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_aesenc_avx_last; + } + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y8))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_aesenc_avx_last: + y5 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y8))); + rsp = (word64)(rsp - 16); + rcx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y5)); +L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(rsi, rbx) = (byte)((byte)r13); + WC_S8(rsp, rcx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_loop; + } + r13 = (word64)(0); + if (((word32)rcx) == (0x10)) { + goto L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_finish_enc; + } +L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_byte_loop: + WC_S8(rsp, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) < (0x10)) { + goto L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_byte_loop; + } +L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_finish_enc: + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + rsp = (word64)(rsp + 16); + y5 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y5)); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + x15 = _mm256_castsi256_si128(y10); +L_AES_GCM_encrypt_vaes_aesenc_last15_enc_avx_done: +L_AES_GCM_encrypt_vaes_done_enc: + rdx = (word32)((word32)r9); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + y0 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rdx)); + y1 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rcx)); + y0 = _mm256_zextsi128_si256(_mm_unpacklo_epi64(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y0)); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + x15 = _mm256_castsi256_si128(y10); + x15 = _mm_shuffle_epi8(x15, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 528))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), x15)); + if (((word32)r14) == (0x10)) { + goto L_AES_GCM_encrypt_vaes_store_tag_16; + } + rcx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); +L_AES_GCM_encrypt_vaes_store_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + WC_S8(r8, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)r14)) { + goto L_AES_GCM_encrypt_vaes_store_tag_loop; + } + goto L_AES_GCM_encrypt_vaes_store_tag_done; +L_AES_GCM_encrypt_vaes_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm256_castsi256_si128(y0)); +L_AES_GCM_encrypt_vaes_store_tag_done: + ; +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_GCM_decrypt_vaes(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res) +{ + word64 rdi, rsi, r12, rax, r8, r9, r11, rbx, r14, r15, r10, rbp, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x15; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5, y6 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(), y8 = _mm256_setzero_si256(), + y9 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(), y14 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[80]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + word32 zf35; + word32 zf36; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r8 = (word64)(size_t)tag; + r9 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r14 = (word64)(word64)tbytes; + r15 = (word64)(size_t)key; + r10 = (word64)(word64)nr; + rbp = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 640; + rsp = (word64)(rsp - 592); + y5 = _mm256_zextsi128_si256(_mm_setzero_si128()); + x15 = _mm_setzero_si128(); + zf1 = (word32)rbx; + zf2 = 0xc; + rdx = (word32)((word32)rbx); + if ((zf1) != (zf2)) { + goto L_AES_GCM_decrypt_vaes_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + y5 = _mm256_zextsi128_si256(_mm_loadl_epi64((const __m128i*)WC_PR(rax, 0))); + y5 = _mm256_zextsi128_si256(_mm_insert_epi32(_mm256_castsi256_si128(y5), ( + int)WC_L32(rax, 8), 2)); + y5 = _mm256_zextsi128_si256(_mm_insert_epi32(_mm256_castsi256_si128(y5), ( + int)(word32)rcx, 3)); + /* H = Encrypt X(=0) and T = Encrypt counter */ + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y6))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + zf3 = (word32)r10; + zf4 = 0xb; + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_12_last; + } + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + zf5 = (word32)r10; + zf6 = 0xd; + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_12_last; + } + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_vaes_calc_iv_12_last: + y6 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y4))); + y1 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 528), _mm256_castsi256_si128(y1)); + goto L_AES_GCM_decrypt_vaes_iv_done; +L_AES_GCM_decrypt_vaes_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)))); + zf7 = (word32)r10; + zf8 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_1_aesenc_avx_last; + } + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)))); + zf9 = (word32)r10; + zf10 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_1_aesenc_avx_last; + } + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_vaes_calc_iv_1_aesenc_avx_last: + y6 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + /* Calc counter */ + /* Initialization vector */ + zf11 = (word32)rdx; + zf12 = 0; + rcx = (word64)(0); + if ((zf11) == (zf12)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_decrypt_vaes_calc_iv_16_loop: + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, + rcx))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y7))); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y5), + 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y5))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y5 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y3)); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y5), 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + y5 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y5), 1)); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_done; + } +L_AES_GCM_decrypt_vaes_calc_iv_lt16: + rsp = (word64)(rsp - 16); + y7 = _mm256_zextsi128_si256(_mm_setzero_si128()); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y7)); +L_AES_GCM_decrypt_vaes_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_loop; + } + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + rsp = (word64)(rsp + 16); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y7))); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y5), + 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y5))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y5 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y3)); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y5), 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + y5 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y5), 1)); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); +L_AES_GCM_decrypt_vaes_calc_iv_done: + /* T = Encrypt counter */ + y0 = _mm256_zextsi128_si256(_mm_setzero_si128()); + rdx = (word32)((word32)rdx << 3); + y0 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rdx)); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y0))); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y5), + 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y5), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y5))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y5 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y3)); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y5), 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + y5 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y5), 1)); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y5 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y2))); + y5 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + /* Encrypt counter */ + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y5))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)))); + zf13 = (word32)r10; + zf14 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_2_aesenc_avx_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)))); + zf15 = (word32)r10; + zf16 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_decrypt_vaes_calc_iv_2_aesenc_avx_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_vaes_calc_iv_2_aesenc_avx_last: + y7 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 528), _mm256_castsi256_si128(y7)); +L_AES_GCM_decrypt_vaes_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x80)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_ser; + } + _mm_storeu_si128((__m128i*)WC_PW(rsp, 560), _mm256_castsi256_si128(y6)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 576), _mm256_castsi256_si128(y5)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi64(_mm256_castsi256_si128(y6), 63)); + y7 = _mm256_zextsi128_si256(_mm_slli_epi64(_mm256_castsi256_si128(y6), 1)); + y8 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y8), 8)); + y7 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0xff)); + y6 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y6), 31)); + y6 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_mod2_128, 0)))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y2 = _mm256_zextsi128_si256(x15); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y6)); + /* H ^ 2 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y6), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y6), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm256_castsi256_si128(y1)); + /* H ^ 4 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y3 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm256_castsi256_si128(y3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), _mm256_castsi256_si128(y4)); + /* H ^ 6 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y1), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), _mm256_castsi256_si128(y4)); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), _mm256_castsi256_si128(y4)); + /* H ^ 8 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y3), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), _mm256_castsi256_si128(y4)); + if (((word32)rdx) < (0x100)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_no_ext; + } + /* H ^ 9 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 48))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), _mm256_castsi256_si128(y4)); + /* H ^ 10 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), _mm256_castsi256_si128(y4)); + /* H ^ 11 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), _mm256_castsi256_si128(y4)); + /* H ^ 12 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), _mm256_castsi256_si128(y4)); + /* H ^ 13 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), _mm256_castsi256_si128(y4)); + /* H ^ 14 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), _mm256_castsi256_si128(y4)); + /* H ^ 15 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), _mm256_castsi256_si128(y4)); + /* H ^ 16 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), _mm256_castsi256_si128(y4)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 224)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 192)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 160)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 128)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y10); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y10); +L_AES_GCM_decrypt_vaes_calc_aad_no_ext: + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xffffff00); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_a256; + } +L_AES_GCM_decrypt_vaes_calc_aad_l256: + /* 256 bytes of AAD */ + rbx = (word64)(r12 + rcx); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 288)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + rcx = (word32)((word32)rcx + 0x100); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_l256; + } +L_AES_GCM_decrypt_vaes_calc_aad_a256: + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_a128; + } + /* 128 bytes of AAD */ + rbx = (word64)(r12 + rcx); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + rcx = (word32)((word32)rcx + 0x80); +L_AES_GCM_decrypt_vaes_calc_aad_a128: + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_arem; + } + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); +L_AES_GCM_decrypt_vaes_calc_aad_rem: + /* 16 bytes of AAD */ + rbx = (word64)(r12 + rcx); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rbx, 0))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + x15 = _mm256_castsi256_si128(y10); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_rem; + } +L_AES_GCM_decrypt_vaes_calc_aad_arem: + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 560))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 576))); + goto L_AES_GCM_decrypt_vaes_calc_aad_partial; +L_AES_GCM_decrypt_vaes_calc_aad_ser: + rdx = (word32)((word32)r11); + rdx = (word32)((word32)rdx & 0xfffffff0); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_partial; + } +L_AES_GCM_decrypt_vaes_calc_aad_16_loop: + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, + rcx))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(x15, 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + x15, 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + x15, 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), x15)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + x15 = _mm256_castsi256_si128(y3); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(x15, 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + x15 = _mm_slli_epi32(x15, 1); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + x15 = _mm_or_si128(x15, _mm256_castsi256_si128(y2)); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + x15 = _mm_or_si128(x15, _mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y2)); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_16_loop; + } +L_AES_GCM_decrypt_vaes_calc_aad_partial: + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_done; + } + rsp = (word64)(rsp - 16); + y7 = _mm256_zextsi128_si256(_mm_setzero_si128()); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y7)); +L_AES_GCM_decrypt_vaes_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_vaes_calc_aad_loop; + } + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + rsp = (word64)(rsp + 16); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + /* ghash_gfmul_avx */ + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi32(x15, 0x4e)); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0x4e)); + y3 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + x15, 0x11)); + y0 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + x15, 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), x15)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y6))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y2), 0)); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y3))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + x15 = _mm256_castsi256_si128(y3); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 8)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 8)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y2))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_srli_epi32(x15, 31)); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 1)); + x15 = _mm_slli_epi32(x15, 1); + y2 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y0), + 12)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), 4)); + y1 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y1), 4)); + x15 = _mm_or_si128(x15, _mm256_castsi256_si128(y2)); + y4 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + x15 = _mm_or_si128(x15, _mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 31)); + y1 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 30)); + y2 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y0)); + y1 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y1), 4)); + y0 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y0), + 12)); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 1)); + y3 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 2)); + y0 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y4), 7)); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y3))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y0))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y1))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y4))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y2)); +L_AES_GCM_decrypt_vaes_calc_aad_done: + /* Calculate counter and H */ + y8 = _mm256_zextsi128_si256(_mm_srli_epi64(_mm256_castsi256_si128(y6), 63)); + y7 = _mm256_zextsi128_si256(_mm_slli_epi64(_mm256_castsi256_si128(y6), 1)); + y8 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y8), 8)); + y7 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0xff)); + y6 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y6), 31)); + y5 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y6 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_mod2_128, 0)))); + y5 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_one, 0)))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y5)); + rbx = (word32)(0); + if (((word32)r9) < (0x80)) { + goto L_AES_GCM_decrypt_vaes_done_128; + } + y2 = _mm256_zextsi128_si256(x15); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y6)); + /* H ^ 2 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y6), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y6), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm256_castsi256_si128(y1)); + /* H ^ 4 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y3 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm256_castsi256_si128(y3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), _mm256_castsi256_si128(y4)); + /* H ^ 6 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y1), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), _mm256_castsi256_si128(y4)); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), _mm256_castsi256_si128(y4)); + /* H ^ 8 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y3), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), _mm256_castsi256_si128(y4)); + if (((word32)r9) < (0x100)) { + goto L_AES_GCM_decrypt_vaes_no_ext; + } + /* H ^ 9 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 48))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), _mm256_castsi256_si128(y4)); + /* H ^ 10 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), _mm256_castsi256_si128(y4)); + /* H ^ 11 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), _mm256_castsi256_si128(y4)); + /* H ^ 12 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), _mm256_castsi256_si128(y4)); + /* H ^ 13 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), _mm256_castsi256_si128(y4)); + /* H ^ 14 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), _mm256_castsi256_si128(y4)); + /* H ^ 15 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), _mm256_castsi256_si128(y4)); + /* H ^ 16 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), _mm256_castsi256_si128(y4)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 224)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 192)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 160)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 128)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y10); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y10); +L_AES_GCM_decrypt_vaes_no_ext: + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + if (((word32)r9) < (0x100)) { + goto L_AES_GCM_decrypt_vaes_after_256; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xffffff00); +L_AES_GCM_decrypt_vaes_loop_256: + /* 256 bytes of input */ + rax = (word64)(rdi + rbx); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 288)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 128)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 160)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 192)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 224)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf17 = (word32)r10; + zf18 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_decrypt_vaes_l1_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf19 = (word32)r10; + zf20 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_decrypt_vaes_l1_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_vaes_l1_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + rbx = (word32)((word32)rbx + 0x80); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf21 = (word32)r10; + zf22 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_decrypt_vaes_l2_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf23 = (word32)r10; + zf24 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_decrypt_vaes_l2_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_vaes_l2_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + rbx = (word32)((word32)rbx + 0x80); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_vaes_loop_256; + } +L_AES_GCM_decrypt_vaes_after_256: + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_vaes_after_128; + } + /* 128 bytes of input */ + rax = (word64)(rdi + rbx); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf25 = (word32)r10; + zf26 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_decrypt_vaes_t_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf27 = (word32)r10; + zf28 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_decrypt_vaes_t_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_vaes_t_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + rbx = (word32)((word32)rbx + 0x80); +L_AES_GCM_decrypt_vaes_after_128: + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); +L_AES_GCM_decrypt_vaes_done_128: + rdx = (word32)((word32)r9); + if (((word32)rbx) >= ((word32)rdx)) { + goto L_AES_GCM_decrypt_vaes_done_dec; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_vaes_last_block_done; + } +L_AES_GCM_decrypt_vaes_last_block_start: + y12 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + rbx))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y6)); + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y12), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), x15)); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y8 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 512), _mm256_castsi256_si128(y8)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)))); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 1)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x11)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 8)); + y9 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), 8)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y11))); + y3 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y9))); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)))); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y2), + 0x4e)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)))); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y9), + 0x4e)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y9), _mm256_castsi256_si128(y3)); + zf29 = (word32)r10; + zf30 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_decrypt_vaes_aesenc_gfmul_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)))); + zf31 = (word32)r10; + zf32 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_GCM_decrypt_vaes_aesenc_gfmul_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_vaes_aesenc_gfmul_last: + y7 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y12)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y0))); + _mm_storeu_si128((__m128i*)WC_PW(rsi, rbx), _mm256_castsi256_si128(y7)); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_vaes_last_block_start; + } +L_AES_GCM_decrypt_vaes_last_block_done: + rcx = (word32)((word32)r9); + rdx = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_decrypt_vaes_aesenc_last15_dec_avx_done; + } + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 512))); + y5 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)))); + zf33 = (word32)r10; + zf34 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_GCM_decrypt_vaes_aesenc_last15_dec_avx_aesenc_avx_last; + } + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y8))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)))); + zf35 = (word32)r10; + zf36 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf35) < (sword32)(zf36)) { + goto L_AES_GCM_decrypt_vaes_aesenc_last15_dec_avx_aesenc_avx_last; + } + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y8))); + y5 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_vaes_aesenc_last15_dec_avx_aesenc_avx_last: + y5 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y8))); + rsp = (word64)(rsp - 32); + rcx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y5)); + y0 = _mm256_zextsi128_si256(_mm_setzero_si128()); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); +L_AES_GCM_decrypt_vaes_aesenc_last15_dec_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + WC_S8(rsp, rcx + 16) = (byte)((byte)r13); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(rsi, rbx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_vaes_aesenc_last15_dec_avx_loop; + } + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 16))); + rsp = (word64)(rsp + 32); + y5 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y5), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y5)); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + x15 = _mm256_castsi256_si128(y10); +L_AES_GCM_decrypt_vaes_aesenc_last15_dec_avx_done: +L_AES_GCM_decrypt_vaes_done_dec: + rdx = (word32)((word32)r9); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + y0 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rdx)); + y1 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rcx)); + y0 = _mm256_zextsi128_si256(_mm_unpacklo_epi64(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y0)); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + x15 = _mm256_castsi256_si128(y10); + x15 = _mm_shuffle_epi8(x15, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 528))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), x15)); + if (((word32)r14) == (0x10)) { + goto L_AES_GCM_decrypt_vaes_cmp_tag_16; + } + rsp = (word64)(rsp - 16); + rcx = (word64)(0); + rbx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); +L_AES_GCM_decrypt_vaes_cmp_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(r8, + rcx)) & 0xff); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)((byte)rbx | ( + byte)r13) & 0xff); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)r14)) { + goto L_AES_GCM_decrypt_vaes_cmp_tag_loop; + } + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)(((byte)rbx) == (0)) & 0xff); + rsp = (word64)(rsp + 16); + rcx = (word64)(0); + goto L_AES_GCM_decrypt_vaes_cmp_tag_done; +L_AES_GCM_decrypt_vaes_cmp_tag_16: + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + y0 = _mm256_zextsi128_si256(_mm_cmpeq_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + rdx = (word32)((word32)_mm_movemask_epi8(_mm256_castsi256_si128(y0))); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + rbx = (word32)(0); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)(((word32)rdx) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_vaes_cmp_tag_done: + WC_S32(rbp, 0) = (word32)((word32)rbx); +} + +#ifdef WOLFSSL_AESGCM_STREAM +WC_X64I_TARGET("aes,pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_init_vaes(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, rsp, rdx, rcx = 0, r13 = 0, + r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)ivec; + r11 = (word64)(word32)ibytes; + r8 = (word64)(size_t)h; + r9 = (word64)(size_t)counter; + rax = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_setzero_si128(); + rdx = (word32)((word32)r11); + if (((word32)rdx) != (0xc)) { + goto L_AES_GCM_init_vaes_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_loadl_epi64((const __m128i*)WC_PR(r10, 0)); + x4 = _mm_insert_epi32(x4, (int)WC_L32(r10, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_xor_si128(x4, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + zf1 = (word32)rsi; + zf2 = 0xb; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_init_vaes_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + zf3 = (word32)rsi; + zf4 = 0xd; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_init_vaes_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_vaes_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x6); + x1 = _mm_aesenclast_si128(x1, x6); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + x15 = x1; + goto L_AES_GCM_init_vaes_iv_done; +L_AES_GCM_init_vaes_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf5 = (word32)rsi; + zf6 = 0xb; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_init_vaes_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x8); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf7 = (word32)rsi; + zf8 = 0xd; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_init_vaes_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x8); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_vaes_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x8); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf9 = (word32)rdx; + zf10 = 0; + rcx = (word64)(0); + if ((zf9) == (zf10)) { + goto L_AES_GCM_init_vaes_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_init_vaes_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_init_vaes_calc_iv_16_loop: + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, rcx)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_vaes_calc_iv_16_loop; + } + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_init_vaes_calc_iv_done; + } +L_AES_GCM_init_vaes_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x7 = _mm_setzero_si128(); + r13 = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x7); +L_AES_GCM_init_vaes_calc_iv_loop: + r12 = (word32)((word32)WC_L8(r10, rcx)); + WC_S8(rsp, r13) = (byte)((byte)r12); + rcx = (word32)((word32)rcx + 1); + r13 = (word32)((word32)r13 + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_vaes_calc_iv_loop; + } + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); +L_AES_GCM_init_vaes_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf11 = (word32)rsi; + zf12 = 0xb; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_init_vaes_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x8); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf13 = (word32)rsi; + zf14 = 0xd; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_init_vaes_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x8); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_vaes_calc_iv_2_aesenc_avx_last: + x7 = _mm_aesenclast_si128(x7, x8); + x15 = x7; +L_AES_GCM_init_vaes_iv_done: + _mm_storeu_si128((__m128i*)WC_PW(rax, 0), x15); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x4); +} + +WC_X64I_TARGET("pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_GCM_aad_update_vaes(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, r8, r9, rsp, rdx, rax = 0, rcx = 0; + __m128i x15; + __m256i y0, y1, y2, y3, y4, y5 = _mm256_setzero_si256(), y6, y7, y8, y9, + y10, y11, y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(), y14 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[64]; + + rdi = (word64)(size_t)addt; + rsi = (word64)(word32)abytes; + r8 = (word64)(size_t)tag; + r9 = (word64)(size_t)h; + + rsp = (word64)(size_t)stk + 512; + rsp = (word64)(rsp - 512); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + rdx = (word32)(0); + y8 = _mm256_zextsi128_si256(_mm_srli_epi64(_mm256_castsi256_si128(y6), 63)); + y7 = _mm256_zextsi128_si256(_mm_slli_epi64(_mm256_castsi256_si128(y6), 1)); + y8 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y8), 8)); + y7 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0xff)); + y6 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y6), 31)); + y6 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_mod2_128, 0)))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y2 = _mm256_zextsi128_si256(x15); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y6)); + /* H ^ 2 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y6), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y6), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm256_castsi256_si128(y1)); + /* H ^ 4 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y3 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm256_castsi256_si128(y3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), _mm256_castsi256_si128(y4)); + /* H ^ 6 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y1), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), _mm256_castsi256_si128(y4)); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), _mm256_castsi256_si128(y4)); + /* H ^ 8 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y3), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), _mm256_castsi256_si128(y4)); + if (((word32)rsi) < (0x100)) { + goto L_AES_GCM_aad_update_vaes_no_ext; + } + /* H ^ 9 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 48))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), _mm256_castsi256_si128(y4)); + /* H ^ 10 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), _mm256_castsi256_si128(y4)); + /* H ^ 11 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), _mm256_castsi256_si128(y4)); + /* H ^ 12 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), _mm256_castsi256_si128(y4)); + /* H ^ 13 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), _mm256_castsi256_si128(y4)); + /* H ^ 14 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), _mm256_castsi256_si128(y4)); + /* H ^ 15 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), _mm256_castsi256_si128(y4)); + /* H ^ 16 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), _mm256_castsi256_si128(y4)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 224)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 192)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 160)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 128)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y10); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y10); +L_AES_GCM_aad_update_vaes_no_ext: + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + rax = (word32)((word32)rsi); + rax = (word32)((word32)rax & 0xffffff00); + if (((word32)rdx) >= ((word32)rax)) { + goto L_AES_GCM_aad_update_vaes_after_256; + } +L_AES_GCM_aad_update_vaes_loop_256: + /* 256 bytes of input */ + rcx = (word64)(rdi + rdx); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 288)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 128)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 160)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 192)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 224)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + rdx = (word32)((word32)rdx + 0x100); + if (((word32)rdx) < ((word32)rax)) { + goto L_AES_GCM_aad_update_vaes_loop_256; + } +L_AES_GCM_aad_update_vaes_after_256: + rax = (word32)((word32)rsi); + rax = (word32)((word32)rax & 0xffffff80); + if (((word32)rdx) >= ((word32)rax)) { + goto L_AES_GCM_aad_update_vaes_after_128; + } + /* 128 bytes of input */ + rcx = (word64)(rdi + rdx); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + rdx = (word32)((word32)rdx + 0x80); +L_AES_GCM_aad_update_vaes_after_128: + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + rax = (word32)((word32)rsi); + rax = (word32)((word32)rax & 0xfffffff0); + if (((word32)rdx) >= ((word32)rax)) { + goto L_AES_GCM_aad_update_vaes_done; + } +L_AES_GCM_aad_update_vaes_tail: + /* 16 bytes of input */ + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + rdx))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + x15 = _mm256_castsi256_si128(y10); + rdx = (word32)((word32)rdx + 0x10); + if (((word32)rdx) < ((word32)rax)) { + goto L_AES_GCM_aad_update_vaes_tail; + } +L_AES_GCM_aad_update_vaes_done: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x15); +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_block_vaes(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8; + __m128i x0, x1; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(size_t)counter; + + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x0 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x1); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf1 = (word32)rsi; + zf2 = 0xb; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_block_vaes_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf3 = (word32)rsi; + zf4 = 0xd; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_block_vaes_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_block_vaes_aesenc_block_last: + x0 = _mm_aesenclast_si128(x0, x1); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_ghash_block_vaes(const unsigned char* addt, + unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, rdx; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + + rdi = (word64)(size_t)addt; + rsi = (word64)(size_t)tag; + rdx = (word64)(size_t)h; + + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x4); +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_GCM_encrypt_update_vaes(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, r13 = 0, r15 = 0, + rcx = 0, rdx = 0; + __m128i x15; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), y6, y7, + y8, y9 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(), y14 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[68]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 544; + rsp = (word64)(rsp - 528); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, 0))); + y8 = _mm256_zextsi128_si256(_mm_srli_epi64(_mm256_castsi256_si128(y6), 63)); + y7 = _mm256_zextsi128_si256(_mm_slli_epi64(_mm256_castsi256_si128(y6), 1)); + y8 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y8), 8)); + y7 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0xff)); + y6 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y6), 31)); + y6 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_mod2_128, 0)))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + r14 = (word32)(0); + if (((word32)r8) < (0x80)) { + goto L_AES_GCM_encrypt_update_vaes_done_128; + } + y2 = _mm256_zextsi128_si256(x15); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y6)); + /* H ^ 2 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y6), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y6), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm256_castsi256_si128(y1)); + /* H ^ 4 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y3 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm256_castsi256_si128(y3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), _mm256_castsi256_si128(y4)); + /* H ^ 6 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y1), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), _mm256_castsi256_si128(y4)); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), _mm256_castsi256_si128(y4)); + /* H ^ 8 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y3), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), _mm256_castsi256_si128(y4)); + if (((word32)r8) < (0x100)) { + goto L_AES_GCM_encrypt_update_vaes_no_ext; + } + /* H ^ 9 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 48))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), _mm256_castsi256_si128(y4)); + /* H ^ 10 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), _mm256_castsi256_si128(y4)); + /* H ^ 11 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), _mm256_castsi256_si128(y4)); + /* H ^ 12 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), _mm256_castsi256_si128(y4)); + /* H ^ 13 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), _mm256_castsi256_si128(y4)); + /* H ^ 14 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), _mm256_castsi256_si128(y4)); + /* H ^ 15 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), _mm256_castsi256_si128(y4)); + /* H ^ 16 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), _mm256_castsi256_si128(y4)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 224)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 192)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 160)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 128)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y10); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y10); +L_AES_GCM_encrypt_update_vaes_no_ext: + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + if (((word32)r8) < (0x100)) { + goto L_AES_GCM_encrypt_update_vaes_after_256; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xffffff00); +L_AES_GCM_encrypt_update_vaes_loop_256: + /* 256 bytes of input */ + r15 = (word64)(r10 + r14); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf1 = (word32)rsi; + zf2 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_update_vaes_p1_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf3 = (word32)rsi; + zf4 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_update_vaes_p1_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_vaes_p1_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + r14 = (word32)((word32)r14 + 0x80); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf5 = (word32)rsi; + zf6 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_encrypt_update_vaes_p2_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf7 = (word32)rsi; + zf8 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_encrypt_update_vaes_p2_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_vaes_p2_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + r14 = (word32)((word32)r14 + 0x80); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 288)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 128)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 160)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 192)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 224)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_vaes_loop_256; + } +L_AES_GCM_encrypt_update_vaes_after_256: + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_vaes_after_128; + } + /* 128 bytes of input */ + r15 = (word64)(r10 + r14); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf9 = (word32)rsi; + zf10 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_encrypt_update_vaes_8_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf11 = (word32)rsi; + zf12 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_encrypt_update_vaes_8_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_vaes_8_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + r14 = (word32)((word32)r14 + 0x80); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); +L_AES_GCM_encrypt_update_vaes_after_128: + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); +L_AES_GCM_encrypt_update_vaes_done_128: + rdx = (word32)((word32)r8); + if (((word32)r14) >= ((word32)rdx)) { + goto L_AES_GCM_encrypt_update_vaes_done_enc; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_vaes_last_block_done; + } + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y8 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm256_castsi256_si128(y8)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)))); + zf13 = (word32)rsi; + zf14 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_update_vaes_aesenc_block_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)))); + zf15 = (word32)rsi; + zf16 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_encrypt_update_vaes_aesenc_block_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_vaes_aesenc_block_last: + y7 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r11, + r14))); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), _mm256_castsi256_si128(y7)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_vaes_last_block_ghash; + } +L_AES_GCM_encrypt_update_vaes_last_block_start: + y12 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r11, + r14))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y8 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm256_castsi256_si128(y8)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)))); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 8)); + y9 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), 8)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y11))); + y3 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y9))); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)))); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y2), + 0x4e)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)))); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y9), + 0x4e)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y9), _mm256_castsi256_si128(y3)); + zf17 = (word32)rsi; + zf18 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_encrypt_update_vaes_aesenc_gfmul_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)))); + zf19 = (word32)rsi; + zf20 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_encrypt_update_vaes_aesenc_gfmul_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_vaes_aesenc_gfmul_last: + y7 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y12)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y0))); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), _mm256_castsi256_si128(y7)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + r14 = (word32)((word32)r14 + 0x10); + x15 = _mm_xor_si128(x15, _mm256_castsi256_si128(y7)); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_vaes_last_block_start; + } +L_AES_GCM_encrypt_update_vaes_last_block_ghash: + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x15, + _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + x15 = _mm256_castsi256_si128(y10); +L_AES_GCM_encrypt_update_vaes_last_block_done: +L_AES_GCM_encrypt_update_vaes_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x15); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_final_vaes(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr) +{ + word64 rdi, rsi, rax, r10, r11, r9, r8, rsp, rdx, rcx, r13 = 0; + __m128i x0, x1, x4, x5, x6, x7, x8, x9, x10, x11; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rax = (word64)(word32)tbytes; + r10 = (word64)(word32)nbytes; + r11 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + r8 = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_srli_epi64(x5, 63); + x7 = _mm_slli_epi64(x5, 1); + x8 = _mm_bslli_si128(x8, 8); + x7 = _mm_or_si128(x7, x8); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x7); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x1 = _mm_cvtsi64_si128((long long)rcx); + x0 = _mm_unpacklo_epi64(x0, x1); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_red_avx */ + x7 = _mm_clmulepi64_si128(x4, x5, 0); + x8 = _mm_clmulepi64_si128(x4, x5, 1); + x9 = _mm_clmulepi64_si128(x4, x5, 0x10); + x10 = _mm_clmulepi64_si128(x4, x5, 0x11); + x8 = _mm_xor_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_mod2_128, 0)); + x11 = _mm_clmulepi64_si128(x9, x7, 1); + x7 = _mm_shuffle_epi32(x7, 0x4e); + x8 = _mm_xor_si128(x8, x11); + x8 = _mm_xor_si128(x8, x7); + x11 = _mm_clmulepi64_si128(x9, x8, 1); + x8 = _mm_shuffle_epi32(x8, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x10 = _mm_xor_si128(x10, x8); + x4 = x10; + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x4, x6); + if (((word32)rax) == (0x10)) { + goto L_AES_GCM_encrypt_final_vaes_store_tag_16; + } + rcx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_final_vaes_store_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + WC_S8(rsi, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)rax)) { + goto L_AES_GCM_encrypt_final_vaes_store_tag_loop; + } + goto L_AES_GCM_encrypt_final_vaes_store_tag_done; +L_AES_GCM_encrypt_final_vaes_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); +L_AES_GCM_encrypt_final_vaes_store_tag_done: + ; +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_GCM_decrypt_update_vaes(const unsigned char* key, int nr, + unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, r13 = 0, rbx = 0, + rcx = 0, rdx = 0; + __m128i x15; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), y6, y7, + y8, y9 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(), y14 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[68]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 544; + rsp = (word64)(rsp - 528); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, 0))); + y8 = _mm256_zextsi128_si256(_mm_srli_epi64(_mm256_castsi256_si128(y6), 63)); + y7 = _mm256_zextsi128_si256(_mm_slli_epi64(_mm256_castsi256_si128(y6), 1)); + y8 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y8), 8)); + y7 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y6), + 0xff)); + y6 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y6), 31)); + y6 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y6), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_mod2_128, 0)))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + r14 = (word32)(0); + if (((word32)r8) < (0x80)) { + goto L_AES_GCM_decrypt_update_vaes_done_128; + } + y2 = _mm256_zextsi128_si256(x15); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y6)); + /* H ^ 2 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y6), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y6), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y6), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y1 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm256_castsi256_si128(y1)); + /* H ^ 4 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y3 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm256_castsi256_si128(y3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), _mm256_castsi256_si128(y4)); + /* H ^ 6 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y1), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), _mm256_castsi256_si128(y4)); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y1), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y1), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), _mm256_castsi256_si128(y4)); + /* H ^ 8 */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y3), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), _mm256_castsi256_si128(y4)); + if (((word32)r8) < (0x100)) { + goto L_AES_GCM_decrypt_update_vaes_no_ext; + } + /* H ^ 9 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 48))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), _mm256_castsi256_si128(y4)); + /* H ^ 10 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), _mm256_castsi256_si128(y4)); + /* H ^ 11 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), _mm256_castsi256_si128(y4)); + /* H ^ 12 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), _mm256_castsi256_si128(y4)); + /* H ^ 13 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), _mm256_castsi256_si128(y4)); + /* H ^ 14 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), _mm256_castsi256_si128(y4)); + /* H ^ 15 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0)); + y8 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 1)); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), _mm256_castsi256_si128(y4)); + /* H ^ 16 */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + y7 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y0), 0)); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y0), _mm256_castsi256_si128(y0), 0x11)); + y8 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y9 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y7), 1)); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y7), + 0x4e)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y11))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y8), 1)); + y8 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x4e)); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), _mm256_castsi256_si128(y4)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 224)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 192)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 160)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 128)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y10); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y10); +L_AES_GCM_decrypt_update_vaes_no_ext: + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + if (((word32)r8) < (0x100)) { + goto L_AES_GCM_decrypt_update_vaes_after_256; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xffffff00); +L_AES_GCM_decrypt_update_vaes_loop_256: + /* 256 bytes of input */ + rbx = (word64)(r11 + r14); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 288)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf1 = (word32)rsi; + zf2 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_decrypt_update_vaes_l1_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf3 = (word32)rsi; + zf4 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_update_vaes_l1_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_vaes_l1_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + r14 = (word32)((word32)r14 + 0x80); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf5 = (word32)rsi; + zf6 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_decrypt_update_vaes_l2_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf7 = (word32)rsi; + zf8 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_decrypt_update_vaes_l2_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_vaes_l2_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + r14 = (word32)((word32)r14 + 0x80); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_vaes_loop_256; + } +L_AES_GCM_decrypt_update_vaes_after_256: + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 96)); + y7 = _mm256_permute4x64_epi64(y7, 0x4e); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 64)); + y8 = _mm256_permute4x64_epi64(y8, 0x4e); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 32)); + y9 = _mm256_permute4x64_epi64(y9, 0x4e); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 0)); + y10 = _mm256_permute4x64_epi64(y10, 0x4e); + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_update_vaes_after_128; + } + /* 128 bytes of input */ + rbx = (word64)(r11 + r14); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + y4 = _mm256_setzero_si256(); + y4 = _mm256_inserti128_si256(y4, x15, 0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_xor_si256(y5, y4); + y0 = _mm256_clmulepi64_epi128(y5, y7, 0); + y1 = _mm256_clmulepi64_epi128(y5, y7, 1); + y2 = _mm256_clmulepi64_epi128(y5, y7, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y7, 0x11); + y11 = y0; + y12 = _mm256_xor_si256(y2, y1); + y13 = y3; + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y8, 0); + y1 = _mm256_clmulepi64_epi128(y5, y8, 1); + y2 = _mm256_clmulepi64_epi128(y5, y8, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y8, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y9, 0); + y1 = _mm256_clmulepi64_epi128(y5, y9, 1); + y2 = _mm256_clmulepi64_epi128(y5, y9, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y9, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y5 = _mm256_shuffle_epi8(y5, y6); + y0 = _mm256_clmulepi64_epi128(y5, y10, 0); + y1 = _mm256_clmulepi64_epi128(y5, y10, 1); + y2 = _mm256_clmulepi64_epi128(y5, y10, 0x10); + y3 = _mm256_clmulepi64_epi128(y5, y10, 0x11); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y5 = _mm256_clmulepi64_epi128(y14, y11, 1); + y11 = _mm256_shuffle_epi32(y11, 0x4e); + y12 = _mm256_xor_si256(y12, y5); + y12 = _mm256_xor_si256(y12, y11); + y5 = _mm256_clmulepi64_epi128(y14, y12, 1); + y12 = _mm256_shuffle_epi32(y12, 0x4e); + y13 = _mm256_xor_si256(y13, y5); + y13 = _mm256_xor_si256(y13, y12); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y13, 1)); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y13), _mm256_castsi256_si128( + y0)); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_epi64, 0))); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + y0 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y0, 0))); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y1, 0))); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y2, 0))); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR( + L_vaes_aes_gcm_inc_y3, 0))); + y3 = _mm256_shuffle_epi8(y3, y6); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_eight, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm256_castsi256_si128(y7)); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y4); + y2 = _mm256_xor_si256(y2, y4); + y3 = _mm256_xor_si256(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf9 = (word32)rsi; + zf10 = 0xb; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_decrypt_update_vaes_t_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + zf11 = (word32)rsi; + zf12 = 0xd; + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_decrypt_update_vaes_t_vaes_ctr8_last; + } + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + y0 = _mm256_aesenc_epi128(y0, y4); + y1 = _mm256_aesenc_epi128(y1, y4); + y2 = _mm256_aesenc_epi128(y2, y4); + y3 = _mm256_aesenc_epi128(y3, y4); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_vaes_t_vaes_ctr8_last: + y0 = _mm256_aesenclast_epi128(y0, y4); + y1 = _mm256_aesenclast_epi128(y1, y4); + y2 = _mm256_aesenclast_epi128(y2, y4); + y3 = _mm256_aesenclast_epi128(y3, y4); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y0 = _mm256_xor_si256(y0, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y2 = _mm256_xor_si256(y2, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y3 = _mm256_xor_si256(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + r14 = (word32)((word32)r14 + 0x80); +L_AES_GCM_decrypt_update_vaes_after_128: + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); +L_AES_GCM_decrypt_update_vaes_done_128: + rdx = (word32)((word32)r8); + if (((word32)r14) >= ((word32)rdx)) { + goto L_AES_GCM_decrypt_update_vaes_done_dec; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_update_vaes_last_block_done; + } +L_AES_GCM_decrypt_update_vaes_last_block_start: + y12 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r11, + r14))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y6)); + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y12), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_mask, 0)))); + y1 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), x15)); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + y7 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_bswap_epi64, 0)))); + y8 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm256_castsi256_si128(y8)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)))); + y9 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 1)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)))); + y11 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y1), _mm256_castsi256_si128(y0), 0)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)))); + y1 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y0), 0x11)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y2 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 8)); + y9 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), 8)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y11))); + y3 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y9))); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)))); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y2), + 0x4e)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y10 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y9), _mm256_castsi256_si128(y0), 0x10)); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)))); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y9), + 0x4e)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + x15 = _mm_xor_si128(_mm256_castsi256_si128(y9), _mm256_castsi256_si128(y3)); + zf13 = (word32)rsi; + zf14 = 0xb; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_decrypt_update_vaes_aesenc_gfmul_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)))); + zf15 = (word32)rsi; + zf16 = 0xd; + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_decrypt_update_vaes_aesenc_gfmul_last; + } + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y7 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y7), + _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)))); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_vaes_aesenc_gfmul_last: + y7 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y12)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y0))); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), _mm256_castsi256_si128(y7)); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_vaes_last_block_start; + } +L_AES_GCM_decrypt_update_vaes_last_block_done: +L_AES_GCM_decrypt_update_vaes_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x15); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_decrypt_final_vaes(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res) +{ + word64 rdi, rsi, rax, r10, r11, r9, r8, rbp, rsp, rdx, rcx, r12 = 0, + r13 = 0; + __m128i x0, x1, x5, x6, x7, x8, x9, x10, x11, x15; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rax = (word64)(word32)tbytes; + r10 = (word64)(word32)nbytes; + r11 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + r8 = (word64)(size_t)initCtr; + rbp = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_srli_epi64(x5, 63); + x7 = _mm_slli_epi64(x5, 1); + x8 = _mm_bslli_si128(x8, 8); + x7 = _mm_or_si128(x7, x8); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x7); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x1 = _mm_cvtsi64_si128((long long)rcx); + x0 = _mm_unpacklo_epi64(x0, x1); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_red_avx */ + x7 = _mm_clmulepi64_si128(x6, x5, 0); + x8 = _mm_clmulepi64_si128(x6, x5, 1); + x9 = _mm_clmulepi64_si128(x6, x5, 0x10); + x10 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_xor_si128(x8, x9); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_gcm_mod2_128, 0)); + x11 = _mm_clmulepi64_si128(x9, x7, 1); + x7 = _mm_shuffle_epi32(x7, 0x4e); + x8 = _mm_xor_si128(x8, x11); + x8 = _mm_xor_si128(x8, x7); + x11 = _mm_clmulepi64_si128(x9, x8, 1); + x8 = _mm_shuffle_epi32(x8, 0x4e); + x10 = _mm_xor_si128(x10, x11); + x10 = _mm_xor_si128(x10, x8); + x6 = x10; + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x6, x15); + if (((word32)rax) == (0x10)) { + goto L_AES_GCM_decrypt_final_vaes_cmp_tag_16; + } + rsp = (word64)(rsp - 16); + rcx = (word64)(0); + r12 = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_final_vaes_cmp_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsi, + rcx)) & 0xff); + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)((byte)r12 | ( + byte)r13) & 0xff); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)rax)) { + goto L_AES_GCM_decrypt_final_vaes_cmp_tag_loop; + } + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)(((byte)r12) == (0)) & 0xff); + rsp = (word64)(rsp + 16); + rcx = (word64)(0); + goto L_AES_GCM_decrypt_final_vaes_cmp_tag_done; +L_AES_GCM_decrypt_final_vaes_cmp_tag_16: + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_cmpeq_epi8(x0, x1); + rdx = (word32)((word32)_mm_movemask_epi8(x0)); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + r12 = (word32)(0); + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)(((word32)rdx) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_final_vaes_cmp_tag_done: + WC_S32(rbp, 0) = (word32)((word32)r12); +} + +#endif /* WOLFSSL_AESGCM_STREAM */ +#ifdef WOLFSSL_AESGCM_SIV +XALIGNED(16) static const word64 L_aes_gcm_siv_bswap_mask_vaes[] + WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +WC_X64I_TARGET("pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_GCMSIV_polyval_vaes(unsigned char* s, + const unsigned char* h, const unsigned char* data, unsigned int blocks) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10 = 0; + __m128i x0, x1; + __m256i y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, y15; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)h; + rdx = (word64)(size_t)data; + rcx = (word64)(word32)blocks; + + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_siv_bswap_mask_vaes, 0))); + y14 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x1, x1, 0)); + y15 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x1, x1, 0x11)); + y13 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x1, x1, 0x10)); + y6 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x1, x1, 1)); + y13 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y13), + _mm256_castsi256_si128(y6))); + y6 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y13), + 8)); + y13 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y13), + 8)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y13))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y2 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y14))); + y2 = _mm256_inserti128_si256(y2, x1, 1); + y14 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), _mm256_castsi256_si128(y2), 0)); + y15 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), _mm256_castsi256_si128(y2), 0x11)); + y13 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), _mm256_castsi256_si128(y2), 0x10)); + y6 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y2), 1)); + y13 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y13), + _mm256_castsi256_si128(y6))); + y6 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y13), + 8)); + y13 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y13), + 8)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y13))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y3 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y14))); + y14 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), x1, 0)); + y15 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), x1, 0x11)); + y13 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y2), x1, 0x10)); + y6 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y2), + x1, 1)); + y13 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y13), + _mm256_castsi256_si128(y6))); + y6 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y13), + 8)); + y13 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y13), + 8)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y13))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y12 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y14))); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y12), 1); + y14 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y2), 0)); + y15 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y2), 0x11)); + y13 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y2), 0x10)); + y6 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y2), 1)); + y13 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y13), + _mm256_castsi256_si128(y6))); + y6 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y13), + 8)); + y13 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y13), + 8)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y13))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y4 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y14))); + y14 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), x1, 0)); + y15 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), x1, 0x11)); + y13 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), x1, 0x10)); + y6 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + x1, 1)); + y13 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y13), + _mm256_castsi256_si128(y6))); + y6 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y13), + 8)); + y13 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y13), + 8)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y13))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y13 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y14))); + y4 = _mm256_inserti128_si256(y4, _mm256_castsi256_si128(y13), 1); + y14 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0)); + y15 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0x11)); + y13 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y3), 0x10)); + y6 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y3), 1)); + y13 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y13), + _mm256_castsi256_si128(y6))); + y6 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y13), + 8)); + y13 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y13), + 8)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y13))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y14))); + y14 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y12), 0)); + y15 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y12), 0x11)); + y13 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128( + y3), _mm256_castsi256_si128(y12), 0x10)); + y6 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y12), 1)); + y13 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y13), + _mm256_castsi256_si128(y6))); + y6 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y13), + 8)); + y13 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y13), + 8)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y13))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y13 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y14))); + y5 = _mm256_inserti128_si256(y5, _mm256_castsi256_si128(y13), 1); + r8 = (word32)((word32)rcx); + r8 = (word32)((word32)r8 << 4); + rax = (word32)(0); + r9 = (word32)((word32)r8); + r9 = (word32)((word32)r9 & 0xffffff80); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_GCMSIV_polyval_vaes_eight_done; + } +L_AES_GCMSIV_polyval_vaes_eight: + r10 = (word64)(rdx + rax); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y12 = _mm256_setzero_si256(); + y12 = _mm256_inserti128_si256(y12, x0, 0); + y6 = _mm256_xor_si256(y6, y12); + y10 = _mm256_clmulepi64_epi128(y6, y5, 0); + y11 = _mm256_clmulepi64_epi128(y6, y5, 0x11); + y14 = _mm256_clmulepi64_epi128(y6, y5, 0x10); + y15 = _mm256_clmulepi64_epi128(y6, y5, 1); + y14 = _mm256_xor_si256(y14, y15); + y15 = _mm256_bslli_epi128(y14, 8); + y14 = _mm256_bsrli_epi128(y14, 8); + y10 = _mm256_xor_si256(y10, y15); + y11 = _mm256_xor_si256(y11, y14); + y12 = _mm256_clmulepi64_epi128(y7, y4, 0); + y13 = _mm256_clmulepi64_epi128(y7, y4, 0x11); + y14 = _mm256_clmulepi64_epi128(y7, y4, 0x10); + y15 = _mm256_clmulepi64_epi128(y7, y4, 1); + y14 = _mm256_xor_si256(y14, y15); + y15 = _mm256_bslli_epi128(y14, 8); + y14 = _mm256_bsrli_epi128(y14, 8); + y12 = _mm256_xor_si256(y12, y15); + y13 = _mm256_xor_si256(y13, y14); + y10 = _mm256_xor_si256(y10, y12); + y11 = _mm256_xor_si256(y11, y13); + y12 = _mm256_clmulepi64_epi128(y8, y3, 0); + y13 = _mm256_clmulepi64_epi128(y8, y3, 0x11); + y14 = _mm256_clmulepi64_epi128(y8, y3, 0x10); + y15 = _mm256_clmulepi64_epi128(y8, y3, 1); + y14 = _mm256_xor_si256(y14, y15); + y15 = _mm256_bslli_epi128(y14, 8); + y14 = _mm256_bsrli_epi128(y14, 8); + y12 = _mm256_xor_si256(y12, y15); + y13 = _mm256_xor_si256(y13, y14); + y10 = _mm256_xor_si256(y10, y12); + y11 = _mm256_xor_si256(y11, y13); + y12 = _mm256_clmulepi64_epi128(y9, y2, 0); + y13 = _mm256_clmulepi64_epi128(y9, y2, 0x11); + y14 = _mm256_clmulepi64_epi128(y9, y2, 0x10); + y15 = _mm256_clmulepi64_epi128(y9, y2, 1); + y14 = _mm256_xor_si256(y14, y15); + y15 = _mm256_bslli_epi128(y14, 8); + y14 = _mm256_bsrli_epi128(y14, 8); + y12 = _mm256_xor_si256(y12, y15); + y13 = _mm256_xor_si256(y13, y14); + y10 = _mm256_xor_si256(y10, y12); + y11 = _mm256_xor_si256(y11, y13); + y12 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y10, 1)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y12))); + y12 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y11, 1)); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y12))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + x0 = _mm_xor_si128(_mm256_castsi256_si128(y15), _mm256_castsi256_si128( + y14)); + rax = (word32)((word32)rax + 0x80); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_GCMSIV_polyval_vaes_eight; + } +L_AES_GCMSIV_polyval_vaes_eight_done: + r9 = (word32)((word32)r8); + r9 = (word32)((word32)r9 & 0xffffffc0); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_GCMSIV_polyval_vaes_four_done; + } +L_AES_GCMSIV_polyval_vaes_four: + r10 = (word64)(rdx + rax); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y12 = _mm256_setzero_si256(); + y12 = _mm256_inserti128_si256(y12, x0, 0); + y6 = _mm256_xor_si256(y6, y12); + y10 = _mm256_clmulepi64_epi128(y6, y3, 0); + y11 = _mm256_clmulepi64_epi128(y6, y3, 0x11); + y14 = _mm256_clmulepi64_epi128(y6, y3, 0x10); + y15 = _mm256_clmulepi64_epi128(y6, y3, 1); + y14 = _mm256_xor_si256(y14, y15); + y15 = _mm256_bslli_epi128(y14, 8); + y14 = _mm256_bsrli_epi128(y14, 8); + y10 = _mm256_xor_si256(y10, y15); + y11 = _mm256_xor_si256(y11, y14); + y12 = _mm256_clmulepi64_epi128(y7, y2, 0); + y13 = _mm256_clmulepi64_epi128(y7, y2, 0x11); + y14 = _mm256_clmulepi64_epi128(y7, y2, 0x10); + y15 = _mm256_clmulepi64_epi128(y7, y2, 1); + y14 = _mm256_xor_si256(y14, y15); + y15 = _mm256_bslli_epi128(y14, 8); + y14 = _mm256_bsrli_epi128(y14, 8); + y12 = _mm256_xor_si256(y12, y15); + y13 = _mm256_xor_si256(y13, y14); + y10 = _mm256_xor_si256(y10, y12); + y11 = _mm256_xor_si256(y11, y13); + y12 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y10, 1)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y12))); + y12 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y11, 1)); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y12))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + x0 = _mm_xor_si128(_mm256_castsi256_si128(y15), _mm256_castsi256_si128( + y14)); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_GCMSIV_polyval_vaes_four; + } +L_AES_GCMSIV_polyval_vaes_four_done: + r9 = (word32)((word32)r8); + r9 = (word32)((word32)r9 & 0xffffffe0); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_GCMSIV_polyval_vaes_pair_done; + } +L_AES_GCMSIV_polyval_vaes_pair: + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, rax)); + y12 = _mm256_setzero_si256(); + y12 = _mm256_inserti128_si256(y12, x0, 0); + y6 = _mm256_xor_si256(y6, y12); + y10 = _mm256_clmulepi64_epi128(y6, y2, 0); + y11 = _mm256_clmulepi64_epi128(y6, y2, 0x11); + y14 = _mm256_clmulepi64_epi128(y6, y2, 0x10); + y15 = _mm256_clmulepi64_epi128(y6, y2, 1); + y14 = _mm256_xor_si256(y14, y15); + y15 = _mm256_bslli_epi128(y14, 8); + y14 = _mm256_bsrli_epi128(y14, 8); + y10 = _mm256_xor_si256(y10, y15); + y11 = _mm256_xor_si256(y11, y14); + y12 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y10, 1)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y10), + _mm256_castsi256_si128(y12))); + y12 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y11, 1)); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y12))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + x0 = _mm_xor_si128(_mm256_castsi256_si128(y15), _mm256_castsi256_si128( + y14)); + rax = (word32)((word32)rax + 0x20); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_GCMSIV_polyval_vaes_pair; + } +L_AES_GCMSIV_polyval_vaes_pair_done: + if (((word32)rax) == ((word32)r8)) { + goto L_AES_GCMSIV_polyval_vaes_done; + } + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, rax))); + y14 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x0, x1, 0)); + y15 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x0, x1, 0x11)); + y12 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x0, x1, 0x10)); + y6 = _mm256_zextsi128_si256(_mm_clmulepi64_si128(x0, x1, 1)); + y12 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y6))); + y6 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y12), + 8)); + y12 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y12), + 8)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + y15 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y12))); + y9 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y15), + 31)); + y14 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 1)); + y15 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y15), + 1)); + y11 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 12)); + y10 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y10), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), 4)); + y14 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y10))); + y15 = _mm256_zextsi128_si256(_mm_or_si128(_mm256_castsi256_si128(y15), + _mm256_castsi256_si128(y11))); + y9 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 31)); + y10 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 30)); + y11 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y14), + 25)); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y10))); + y9 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y9), + _mm256_castsi256_si128(y11))); + y10 = _mm256_zextsi128_si256(_mm_bsrli_si128(_mm256_castsi256_si128(y9), + 4)); + y9 = _mm256_zextsi128_si256(_mm_bslli_si128(_mm256_castsi256_si128(y9), + 12)); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y9))); + y6 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 1)); + y7 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 2)); + y8 = _mm256_zextsi128_si256(_mm_srli_epi32(_mm256_castsi256_si128(y14), 7)); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y7))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y6 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y10))); + y14 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y14), + _mm256_castsi256_si128(y6))); + x0 = _mm_xor_si128(_mm256_castsi256_si128(y15), _mm256_castsi256_si128( + y14)); +L_AES_GCMSIV_polyval_vaes_done: + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_siv_bswap_mask_vaes, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); +} + +XALIGNED(32) static const word64 L_aes_gcmsiv_ctr_inc_vaes[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0x0000000000000002ULL, 0x0000000000000000ULL, + 0x0000000000000003ULL, 0x0000000000000000ULL, + 0x0000000000000004ULL, 0x0000000000000000ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000000000008ULL, 0x0000000000000000ULL, + 0x0000000000000009ULL, 0x0000000000000000ULL, + 0x000000000000000aULL, 0x0000000000000000ULL, + 0x000000000000000bULL, 0x0000000000000000ULL, + 0x000000000000000cULL, 0x0000000000000000ULL, + 0x000000000000000dULL, 0x0000000000000000ULL, + 0x000000000000000eULL, 0x0000000000000000ULL, + 0x000000000000000fULL, 0x0000000000000000ULL, + 0x0000000000000010ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("aes,vaes,avx2") +WOLFSSL_LOCAL void AES_GCMSIV_ctr_vaes(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11 = 0, r12 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7, y8, y9, y10, + y11 = _mm256_setzero_si256(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + r9 = (word64)(size_t)ctr; + + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 0))); + y8 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcmsiv_ctr_inc_vaes, 128))); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcmsiv_ctr_inc_vaes, 32))); + y10 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcmsiv_ctr_inc_vaes, 16))); + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x80; + r10 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_GCMSIV_ctr_vaes_done_128; + } + r10 = (word32)((word32)r10 & 0xffffff80); + y4 = _mm256_add_epi32(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_gcmsiv_ctr_inc_vaes, 0))); + y5 = _mm256_add_epi32(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_gcmsiv_ctr_inc_vaes, 32))); + y6 = _mm256_add_epi32(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_gcmsiv_ctr_inc_vaes, 64))); + y7 = _mm256_add_epi32(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_gcmsiv_ctr_inc_vaes, 96))); +L_AES_GCMSIV_ctr_vaes_enc_128: + /* 128 bytes of input */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + y0 = y4; + y1 = y5; + y2 = y6; + y3 = y7; + y4 = _mm256_add_epi32(y4, y8); + y5 = _mm256_add_epi32(y5, y8); + y6 = _mm256_add_epi32(y6, y8); + y7 = _mm256_add_epi32(y7, y8); + /* aes_enc_block */ + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y11); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y11); + y3 = _mm256_xor_si256(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + zf3 = (word32)r8; + zf4 = 0xb; + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCMSIV_ctr_vaes_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + zf5 = (word32)r8; + zf6 = 0xd; + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCMSIV_ctr_vaes_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesenc_epi128(y0, y11); + y1 = _mm256_aesenc_epi128(y1, y11); + y2 = _mm256_aesenc_epi128(y2, y11); + y3 = _mm256_aesenc_epi128(y3, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_GCMSIV_ctr_vaes_128_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y11); + y1 = _mm256_aesenclast_epi128(y1, y11); + y2 = _mm256_aesenclast_epi128(y2, y11); + y3 = _mm256_aesenclast_epi128(y3, y11); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32))); + y2 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64))); + y3 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 96), y3); + rax = (word32)((word32)rax + 0x80); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_vaes_enc_128; + } + y7 = _mm256_permute2x128_si256(y4, y4, 0); +L_AES_GCMSIV_ctr_vaes_done_128: + r10 = (word32)((word32)rdx); + r10 = (word32)((word32)r10 & 0xffffffe0); + if (((word32)rax) == ((word32)r10)) { + goto L_AES_GCMSIV_ctr_vaes_done_32; + } +L_AES_GCMSIV_ctr_vaes_enc_32: + /* 32 bytes of input */ + /* siv_aes_ctr_enc_32 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + y0 = _mm256_add_epi32(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_gcmsiv_ctr_inc_vaes, 0))); + y7 = _mm256_add_epi32(y7, y9); + /* aes_enc_block */ + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesenc_epi128(y0, y11); + zf7 = (word32)r8; + zf8 = 0xb; + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCMSIV_ctr_vaes_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesenc_epi128(y0, y11); + zf9 = (word32)r8; + zf10 = 0xd; + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCMSIV_ctr_vaes_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesenc_epi128(y0, y11); + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_GCMSIV_ctr_vaes_32_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y11); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y0); + rax = (word32)((word32)rax + 0x20); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_vaes_enc_32; + } +L_AES_GCMSIV_ctr_vaes_done_32: + zf11 = (word32)rax; + zf12 = (word32)rdx; + r10 = (word32)((word32)rdx); + if ((zf11) == (zf12)) { + goto L_AES_GCMSIV_ctr_vaes_done_enc; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_GCMSIV_ctr_vaes_enc_16: + /* 16 bytes of input */ + y0 = _mm256_zextsi128_si256(_mm256_castsi256_si128(y7)); + y7 = _mm256_add_epi32(y7, y10); + /* aes_enc_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf13 = (word32)r8; + zf14 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCMSIV_ctr_vaes_16_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf15 = (word32)r8; + zf16 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCMSIV_ctr_vaes_16_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_GCMSIV_ctr_vaes_16_aes_enc_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + r11 = (word64)(rdi + rax); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)))); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), _mm256_castsi256_si128(y0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_vaes_enc_16; + } +L_AES_GCMSIV_ctr_vaes_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm256_castsi256_si128(y7)); +} + +#endif /* WOLFSSL_AESGCM_SIV */ +#endif /* HAVE_INTEL_VAES */ +#ifdef HAVE_INTEL_AVX512 +XALIGNED(32) static const word64 L_avx512_aes_gcm_inc_z0[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000003ULL, +}; + +XALIGNED(32) static const word64 L_avx512_aes_gcm_inc_z1[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, + 0x0000000000000000ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000007ULL, +}; + +XALIGNED(32) static const word64 L_avx512_aes_gcm_inc_z2[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000008ULL, + 0x0000000000000000ULL, 0x0000000000000009ULL, + 0x0000000000000000ULL, 0x000000000000000aULL, + 0x0000000000000000ULL, 0x000000000000000bULL, +}; + +XALIGNED(32) static const word64 L_avx512_aes_gcm_inc_z3[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x000000000000000cULL, + 0x0000000000000000ULL, 0x000000000000000dULL, + 0x0000000000000000ULL, 0x000000000000000eULL, + 0x0000000000000000ULL, 0x000000000000000fULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_sixteen[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000010ULL, +}; + +XALIGNED(16) static const word64 L_GCM_generate_m0_avx512_rev8[] + WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_GCM_generate_m0_avx512_mod2_128[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0xe100000000000000ULL, +}; + +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL void GCM_generate_m0_avx512(const unsigned char* h, + unsigned char* m) +{ + word64 rdi, rsi; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10; + + rdi = (word64)(size_t)h; + rsi = (word64)(size_t)m; + + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_GCM_generate_m0_avx512_rev8, + 0)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR( + L_GCM_generate_m0_avx512_mod2_128, 0)); + x8 = _mm_setzero_si128(); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x8); + x8 = x0; + x0 = _mm_shuffle_epi8(x0, x9); + x5 = _mm_slli_epi64(x0, 63); + x4 = _mm_srli_epi64(x0, 1); + x1 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x1 = _mm_shuffle_epi32(x1, 0xff); + x4 = _mm_or_si128(x4, x5); + x1 = _mm_srai_epi32(x1, 31); + x1 = _mm_and_si128(x1, x10); + x1 = _mm_xor_si128(x1, x4); + x5 = _mm_slli_epi64(x1, 63); + x4 = _mm_srli_epi64(x1, 1); + x2 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x2 = _mm_shuffle_epi32(x2, 0xff); + x4 = _mm_or_si128(x4, x5); + x2 = _mm_srai_epi32(x2, 31); + x2 = _mm_and_si128(x2, x10); + x2 = _mm_xor_si128(x2, x4); + x5 = _mm_slli_epi64(x2, 63); + x4 = _mm_srli_epi64(x2, 1); + x3 = _mm_bslli_si128(x5, 8); + x5 = _mm_bsrli_si128(x5, 8); + x3 = _mm_shuffle_epi32(x3, 0xff); + x4 = _mm_or_si128(x4, x5); + x3 = _mm_srai_epi32(x3, 31); + x3 = _mm_and_si128(x3, x10); + x3 = _mm_xor_si128(x3, x4); + x3 = _mm_shuffle_epi8(x3, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x0 = _mm_shuffle_epi8(x0, x9); + x8 = _mm_xor_si128(x3, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x3); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x1); + x4 = _mm_xor_si128(x3, x1); + x5 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x8, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 128), x0); + x1 = _mm_xor_si128(x1, x0); + x4 = _mm_xor_si128(x3, x0); + x6 = _mm_xor_si128(x2, x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 144), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 160), x6); + x6 = _mm_xor_si128(x3, x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 176), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 192), x1); + x4 = _mm_xor_si128(x3, x1); + x5 = _mm_xor_si128(x2, x1); + x6 = _mm_xor_si128(x8, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 208), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 224), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 240), x6); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 256), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 272), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 288), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 304), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 320), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 336), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 352), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 368), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 384), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 400), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 416), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 432), x3); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 240)); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + x4 = _mm_slli_epi64(x0, 60); + x5 = _mm_slli_epi64(x1, 60); + x6 = _mm_slli_epi64(x2, 60); + x7 = _mm_slli_epi64(x3, 60); + x0 = _mm_srli_epi64(x0, 4); + x1 = _mm_srli_epi64(x1, 4); + x2 = _mm_srli_epi64(x2, 4); + x3 = _mm_srli_epi64(x3, 4); + x4 = _mm_bsrli_si128(x4, 8); + x5 = _mm_bsrli_si128(x5, 8); + x6 = _mm_bsrli_si128(x6, 8); + x7 = _mm_bsrli_si128(x7, 8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + x0 = _mm_shuffle_epi8(x0, x9); + x1 = _mm_shuffle_epi8(x1, x9); + x2 = _mm_shuffle_epi8(x2, x9); + x3 = _mm_shuffle_epi8(x3, x9); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 448), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 464), x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 480), x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 496), x3); +} + +XALIGNED(16) static const word64 L_avx512_aes_gcm_one[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000001ULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_two[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_three[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000003ULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_four[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_five[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000005ULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_six[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000006ULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_seven[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000007ULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_eight[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000008ULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_bswap_epi64[] + WC_X64I_UNUSED = { + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_bswap_mask[] + WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_avx512_aes_gcm_mod2_128[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0xc200000000000000ULL, +}; + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_GCM_encrypt_avx512(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr) +{ + word64 rdi, rsi, r12, rax, r8, r9, r11, rbx, r14, r15, r10, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x4, x5 = _mm_setzero_si128(), x6, + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(); + __m512i z1 = _mm512_setzero_si512(), z2 = _mm512_setzero_si512(), + z3 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), + z12 = _mm512_setzero_si512(), z13 = _mm512_setzero_si512(), + z14 = _mm512_setzero_si512(), z15 = _mm512_setzero_si512(), + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(), + z22 = _mm512_setzero_si512(), z23 = _mm512_setzero_si512(), + z24 = _mm512_setzero_si512(), z25 = _mm512_setzero_si512(), + z26 = _mm512_setzero_si512(), z27 = _mm512_setzero_si512(), + z28 = _mm512_setzero_si512(), z29 = _mm512_setzero_si512(), + z30 = _mm512_setzero_si512(), z31 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[144]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + word32 zf35; + word32 zf36; + word32 zf37; + word32 zf38; + word32 zf39; + word32 zf40; + word32 zf41; + word32 zf42; + word32 zf43; + word32 zf44; + word32 zf45; + word32 zf46; + word32 zf47; + word32 zf48; + word32 zf49; + word32 zf50; + word32 zf51; + word32 zf52; + word32 zf53; + word32 zf54; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r8 = (word64)(size_t)tag; + r9 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r14 = (word64)(word64)tbytes; + r15 = (word64)(size_t)key; + r10 = (word64)(word64)nr; + + rsp = (word64)(size_t)stk + 1152; + rsp = (word64)(rsp - 1120); + x4 = _mm_setzero_si128(); + x6 = _mm_setzero_si128(); + rdx = (word32)((word32)rbx); + if (((word32)rdx) != (0xc)) { + goto L_AES_GCM_encrypt_avx512_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_loadl_epi64((const __m128i*)WC_PR(rax, 0)); + x4 = _mm_insert_epi32(x4, (int)WC_L32(rax, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(x4, x5)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + zf1 = (word32)r10; + zf2 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + zf3 = (word32)r10; + zf4 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_encrypt_avx512_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z1), + x7)); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1040), _mm512_castsi512_si128(z1)); + goto L_AES_GCM_encrypt_avx512_iv_done; +L_AES_GCM_encrypt_avx512_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf5 = (word32)r10; + zf6 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, _mm512_castsi512_si128(z9)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf7 = (word32)r10; + zf8 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, _mm512_castsi512_si128(z9)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, _mm512_castsi512_si128(z9)); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf9 = (word32)rdx; + zf10 = 0; + rcx = (word64)(0); + if ((zf9) == (zf10)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_encrypt_avx512_calc_iv_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rax, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x4, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x4, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x4)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x4 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x4, 31)); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z2)); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_done; + } +L_AES_GCM_encrypt_avx512_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_encrypt_avx512_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x4, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x4, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x4)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x4 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x4, 31)); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z2)); +L_AES_GCM_encrypt_avx512_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x4, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x4, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x4)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x4 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x4, 31)); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z2)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x8 = _mm_xor_si128(x8, x4); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf11 = (word32)r10; + zf12 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf13 = (word32)r10; + zf14 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_avx512_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_calc_iv_2_aesenc_avx_last: + x8 = _mm_aesenclast_si128(x8, _mm512_castsi512_si128(z9)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1040), x8); +L_AES_GCM_encrypt_avx512_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x80)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_ser; + } + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1088), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1104), x4); + z9 = _mm512_zextsi128_si512(_mm_srli_epi64(x5, 63)); + x8 = _mm_slli_epi64(x5, 1); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 8)); + x8 = _mm_or_si128(x8, _mm512_castsi512_si128(z9)); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + z2 = _mm512_zextsi128_si512(x6); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x0 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x0)); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z1 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm512_castsi512_si128(z1)); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z3 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm512_castsi512_si128(z3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z1))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z3), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z3))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z3), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z3), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + /* H ^ 9 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x7); + /* H ^ 10 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x7); + /* H ^ 11 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), x7); + /* H ^ 12 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), x7); + /* H ^ 13 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), x7); + /* H ^ 14 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), x7); + /* H ^ 15 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), x7); + /* H ^ 16 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), x7); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xffffff00); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_a256; + } +L_AES_GCM_encrypt_avx512_calc_aad_l256: + /* 256 bytes of AAD */ + rbx = (word64)(r12 + rcx); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rcx = (word32)((word32)rcx + 0x100); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_l256; + } +L_AES_GCM_encrypt_avx512_calc_aad_a256: + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_a128; + } + /* 128 bytes of AAD */ + rbx = (word64)(r12 + rcx); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rcx = (word32)((word32)rcx + 0x80); +L_AES_GCM_encrypt_avx512_calc_aad_a128: + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_arem; + } + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_encrypt_avx512_calc_aad_rem: + /* 16 bytes of AAD */ + rbx = (word64)(r12 + rcx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 0)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x6)); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x6 = _mm512_castsi512_si128(z11); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_rem; + } +L_AES_GCM_encrypt_avx512_calc_aad_arem: + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1088)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1104)); + goto L_AES_GCM_encrypt_avx512_calc_aad_partial; +L_AES_GCM_encrypt_avx512_calc_aad_ser: + rdx = (word32)((word32)r11); + rdx = (word32)((word32)rdx & 0xfffffff0); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_partial; + } +L_AES_GCM_encrypt_avx512_calc_aad_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x6, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x6)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x6 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x6 = _mm_xor_si128(x6, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x6, 31)); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x6 = _mm_or_si128(x6, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x6 = _mm_xor_si128(x6, _mm512_castsi512_si128(z2)); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_16_loop; + } +L_AES_GCM_encrypt_avx512_calc_aad_partial: + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_done; + } + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_encrypt_avx512_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx512_calc_aad_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x6, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x6)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x6 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x6 = _mm_xor_si128(x6, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x6, 31)); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x6 = _mm_or_si128(x6, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x6 = _mm_xor_si128(x6, _mm512_castsi512_si128(z2)); +L_AES_GCM_encrypt_avx512_calc_aad_done: + /* Calculate counter and H */ + z9 = _mm512_zextsi128_si512(_mm_srli_epi64(x5, 63)); + x8 = _mm_slli_epi64(x5, 1); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 8)); + x8 = _mm_or_si128(x8, _mm512_castsi512_si128(z9)); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_one, 0))); + x5 = _mm_xor_si128(x5, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x4); + rbx = (word32)(0); + if (((word32)r9) < (0x80)) { + goto L_AES_GCM_encrypt_avx512_done_128; + } + z2 = _mm512_zextsi128_si512(x6); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x0 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x0)); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z1 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm512_castsi512_si128(z1)); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z3 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm512_castsi512_si128(z3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z1))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z3), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z3))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z3), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z3), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + if (((word32)r9) < (0x100)) { + goto L_AES_GCM_encrypt_avx512_no_ext2; + } + /* H ^ 9 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x7); + /* H ^ 10 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x7); + /* H ^ 11 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), x7); + /* H ^ 12 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), x7); + /* H ^ 13 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), x7); + /* H ^ 14 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), x7); + /* H ^ 15 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), x7); + /* H ^ 16 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), x7); + if (((word32)r9) < (0x200)) { + goto L_AES_GCM_encrypt_avx512_no_ext; + } + /* H ^ 17 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 128))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 256), x7); + /* H ^ 18 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 272), x7); + /* H ^ 19 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 144))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 288), x7); + /* H ^ 20 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 304), x7); + /* H ^ 21 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 160))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 320), x7); + /* H ^ 22 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 160)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 336), x7); + /* H ^ 23 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 160)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 176))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 352), x7); + /* H ^ 24 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 176)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 368), x7); + /* H ^ 25 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 176)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 192))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 384), x7); + /* H ^ 26 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 192)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 400), x7); + /* H ^ 27 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 192)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 208))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 416), x7); + /* H ^ 28 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 208)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 432), x7); + /* H ^ 29 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 208)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 224))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 448), x7); + /* H ^ 30 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 224)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 464), x7); + /* H ^ 31 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 224)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 240))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 480), x7); + /* H ^ 32 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 240)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 496), x7); +L_AES_GCM_encrypt_avx512_no_ext: +L_AES_GCM_encrypt_avx512_no_ext2: + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + z1 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + z3 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + if (((word32)r9) < (0x200)) { + goto L_AES_GCM_encrypt_avx512_no_windows; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffe00); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 448)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 384)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 320)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 256)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + _mm512_storeu_si512((void*)WC_PW(rsp, 512), z23); + _mm512_storeu_si512((void*)WC_PW(rsp, 576), z24); + _mm512_storeu_si512((void*)WC_PW(rsp, 640), z25); + _mm512_storeu_si512((void*)WC_PW(rsp, 704), z26); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + _mm512_storeu_si512((void*)WC_PW(rsp, 768), z23); + _mm512_storeu_si512((void*)WC_PW(rsp, 832), z24); + _mm512_storeu_si512((void*)WC_PW(rsp, 896), z25); + _mm512_storeu_si512((void*)WC_PW(rsp, 960), z26); + /* 512 bytes of input */ + rcx = (word64)(rsi + rbx); + WC_S64(rsp, 1056) = (word64)(rcx); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf15 = (word32)r10; + zf16 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_encrypt_avx512_p1_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf17 = (word32)r10; + zf18 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_encrypt_avx512_p1_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_p1_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + rbx = (word32)((word32)rbx + 0x100); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf19 = (word32)r10; + zf20 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_encrypt_avx512_p2_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf21 = (word32)r10; + zf22 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_encrypt_avx512_p2_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_p2_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + rbx = (word32)((word32)rbx + 0x100); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_last_win; + } +L_AES_GCM_encrypt_avx512_win_loop: + rcx = (word64)(rsi + rbx); + WC_S64(rsp, 1072) = (word64)(rcx); + r12 = (word64)(WC_L64(rsp, 1056)); + z21 = _mm512_setzero_si512(); + z21 = _mm512_inserti32x4(z21, x6, 0); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z31 = _mm512_loadu_si512((const void*)WC_PR(r12, 0)); + z31 = _mm512_shuffle_epi8(z31, z30); + z31 = _mm512_xor_si512(z31, z21); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0x11); + z27 = z23; + z28 = _mm512_xor_si512(z25, z24); + z29 = z26; + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z31 = _mm512_loadu_si512((const void*)WC_PR(r12, 64)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z31 = _mm512_loadu_si512((const void*)WC_PR(r12, 128)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z31 = _mm512_loadu_si512((const void*)WC_PR(r12, 192)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf23 = (word32)r10; + zf24 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_encrypt_avx512_a_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf25 = (word32)r10; + zf26 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_encrypt_avx512_a_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_a_il_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + rbx = (word32)((word32)rbx + 0x100); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z31 = _mm512_loadu_si512((const void*)WC_PR(r12, 256)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z31 = _mm512_loadu_si512((const void*)WC_PR(r12, 320)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z31 = _mm512_loadu_si512((const void*)WC_PR(r12, 384)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z31 = _mm512_loadu_si512((const void*)WC_PR(r12, 448)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf27 = (word32)r10; + zf28 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_encrypt_avx512_b_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf29 = (word32)r10; + zf30 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_encrypt_avx512_b_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_b_il_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + rbx = (word32)((word32)rbx + 0x100); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z23 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z23, 0x96); + z23 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z23, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rcx = (word64)(WC_L64(rsp, 1072)); + WC_S64(rsp, 1056) = (word64)(rcx); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_win_loop; + } +L_AES_GCM_encrypt_avx512_last_win: + rcx = (word64)(WC_L64(rsp, 1056)); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 512)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 576)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 640)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 704)); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 768)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 832)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 896)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 960)); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 256)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 320)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 384)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 448)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); +L_AES_GCM_encrypt_avx512_no_windows: + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xffffff00); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_after_256; + } + /* 256 bytes of input */ + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf31 = (word32)r10; + zf32 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_GCM_encrypt_avx512_pro_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf33 = (word32)r10; + zf34 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_GCM_encrypt_avx512_pro_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_pro_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + WC_S64(rsp, 1056) = (word64)(rdx); + rbx = (word32)((word32)rbx + 0x100); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_last_ghash; + } +L_AES_GCM_encrypt_avx512_ghash_128: + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf35 = (word32)r10; + zf36 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf35) < (sword32)(zf36)) { + goto L_AES_GCM_encrypt_avx512_pip_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf37 = (word32)r10; + zf38 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf37) < (sword32)(zf38)) { + goto L_AES_GCM_encrypt_avx512_pip_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_pip_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + rcx = (word64)(WC_L64(rsp, 1056)); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + WC_S64(rsp, 1056) = (word64)(rdx); + rbx = (word32)((word32)rbx + 0x100); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_ghash_128; + } +L_AES_GCM_encrypt_avx512_last_ghash: + rcx = (word64)(WC_L64(rsp, 1056)); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); +L_AES_GCM_encrypt_avx512_after_256: + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_after_128; + } + /* 128 bytes of input */ + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + z1 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + z3 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_eight, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + zf39 = (word32)r10; + zf40 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf39) < (sword32)(zf40)) { + goto L_AES_GCM_encrypt_avx512_8_avx512_ctr8_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + zf41 = (word32)r10; + zf42 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf41) < (sword32)(zf42)) { + goto L_AES_GCM_encrypt_avx512_8_avx512_ctr8_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_8_avx512_ctr8_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rbx = (word32)((word32)rbx + 0x80); +L_AES_GCM_encrypt_avx512_after_128: + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_encrypt_avx512_done_128: + rdx = (word32)((word32)r9); + if (((word32)rbx) >= ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx512_done_enc; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_last_block_done; + } + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + x8 = _mm_shuffle_epi8(_mm512_castsi512_si128(z9), _mm_loadu_si128(( + const __m128i*)WC_PR(L_avx512_aes_gcm_bswap_epi64, 0))); + z9 = _mm512_zextsi128_si512(_mm_add_epi32(_mm512_castsi512_si128(z9), + _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), _mm512_castsi512_si128(z9)); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf43 = (word32)r10; + zf44 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf43) < (sword32)(zf44)) { + goto L_AES_GCM_encrypt_avx512_aesenc_block_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf45 = (word32)r10; + zf46 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf45) < (sword32)(zf46)) { + goto L_AES_GCM_encrypt_avx512_aesenc_block_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_aesenc_block_last: + x8 = _mm_aesenclast_si128(x8, _mm512_castsi512_si128(z9)); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + rbx))); + x8 = _mm_xor_si128(x8, _mm512_castsi512_si128(z9)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, rbx), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_last_block_ghash; + } +L_AES_GCM_encrypt_avx512_last_block_start: + z13 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + rbx))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + x8 = _mm_shuffle_epi8(_mm512_castsi512_si128(z9), _mm_loadu_si128(( + const __m128i*)WC_PR(L_avx512_aes_gcm_bswap_epi64, 0))); + z9 = _mm512_zextsi128_si512(_mm_add_epi32(_mm512_castsi512_si128(z9), + _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), _mm512_castsi512_si128(z9)); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + z10 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 1)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 8)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z10), + 8)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z12))); + z3 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z10))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z2), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z2), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z10), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z10), _mm512_castsi512_si128(z3)); + zf47 = (word32)r10; + zf48 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf47) < (sword32)(zf48)) { + goto L_AES_GCM_encrypt_avx512_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf49 = (word32)r10; + zf50 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf49) < (sword32)(zf50)) { + goto L_AES_GCM_encrypt_avx512_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, _mm512_castsi512_si128(z9)); + x0 = _mm512_castsi512_si128(z13); + x8 = _mm_xor_si128(x8, x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, rbx), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + rbx = (word32)((word32)rbx + 0x10); + x6 = _mm_xor_si128(x6, x8); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_encrypt_avx512_last_block_start; + } +L_AES_GCM_encrypt_avx512_last_block_ghash: + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x6)); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x6 = _mm512_castsi512_si128(z11); +L_AES_GCM_encrypt_avx512_last_block_done: + rcx = (word32)((word32)r9); + rdx = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_done; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + x4 = _mm_xor_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf51 = (word32)r10; + zf52 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf51) < (sword32)(zf52)) { + goto L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, _mm512_castsi512_si128(z9)); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf53 = (word32)r10; + zf54 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf53) < (sword32)(zf54)) { + goto L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, _mm512_castsi512_si128(z9)); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_aesenc_avx_last: + x4 = _mm_aesenclast_si128(x4, _mm512_castsi512_si128(z9)); + rsp = (word64)(rsp - 16); + rcx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); +L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(rsi, rbx) = (byte)((byte)r13); + WC_S8(rsp, rcx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_loop; + } + r13 = (word64)(0); + if (((word32)rcx) == (0x10)) { + goto L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_finish_enc; + } +L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_byte_loop: + WC_S8(rsp, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) < (0x10)) { + goto L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_byte_loop; + } +L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_finish_enc: + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x4); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x6)); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x6 = _mm512_castsi512_si128(z11); +L_AES_GCM_encrypt_avx512_aesenc_last15_enc_avx_done: +L_AES_GCM_encrypt_avx512_done_enc: + rdx = (word32)((word32)r9); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + z1 = _mm512_zextsi128_si512(_mm_cvtsi64_si128((long long)rcx)); + x0 = _mm_unpacklo_epi64(x0, _mm512_castsi512_si128(z1)); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x6)); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x6 = _mm512_castsi512_si128(z11); + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1040)); + x0 = _mm_xor_si128(x0, x6); + if (((word32)r14) == (0x10)) { + goto L_AES_GCM_encrypt_avx512_store_tag_16; + } + rcx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_avx512_store_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + WC_S8(r8, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)r14)) { + goto L_AES_GCM_encrypt_avx512_store_tag_loop; + } + goto L_AES_GCM_encrypt_avx512_store_tag_done; +L_AES_GCM_encrypt_avx512_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x0); +L_AES_GCM_encrypt_avx512_store_tag_done: + ; +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_GCM_decrypt_avx512(const unsigned char* in, + unsigned char* out, const unsigned char* addt, const unsigned char* ivec, + const unsigned char* tag, unsigned int nbytes, unsigned int abytes, + unsigned int ibytes, unsigned int tbytes, const unsigned char* key, int nr, + int* res) +{ + word64 rdi, rsi, r12, rax, r8, r9, r11, rbx, r14, r15, r10, rbp, rsp, rdx, + rcx = 0, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x4, x5 = _mm_setzero_si128(), x6, + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(); + __m512i z1 = _mm512_setzero_si512(), z2 = _mm512_setzero_si512(), + z3 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), + z12 = _mm512_setzero_si512(), z13 = _mm512_setzero_si512(), + z14 = _mm512_setzero_si512(), z15 = _mm512_setzero_si512(), + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(), + z22 = _mm512_setzero_si512(), z23 = _mm512_setzero_si512(), + z24 = _mm512_setzero_si512(), z25 = _mm512_setzero_si512(), + z26 = _mm512_setzero_si512(), z27 = _mm512_setzero_si512(), + z28 = _mm512_setzero_si512(), z29 = _mm512_setzero_si512(), + z30 = _mm512_setzero_si512(), z31 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[144]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + word32 zf35; + word32 zf36; + word32 zf37; + word32 zf38; + word32 zf39; + word32 zf40; + word32 zf41; + word32 zf42; + word32 zf43; + word32 zf44; + word32 zf45; + word32 zf46; + word32 zf47; + word32 zf48; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + r12 = (word64)(size_t)addt; + rax = (word64)(size_t)ivec; + r8 = (word64)(size_t)tag; + r9 = (word64)(word64)nbytes; + r11 = (word64)(word64)abytes; + rbx = (word64)(word64)ibytes; + r14 = (word64)(word64)tbytes; + r15 = (word64)(size_t)key; + r10 = (word64)(word64)nr; + rbp = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 1152; + rsp = (word64)(rsp - 1120); + x4 = _mm_setzero_si128(); + x6 = _mm_setzero_si128(); + zf1 = (word32)rbx; + zf2 = 0xc; + rdx = (word32)((word32)rbx); + if ((zf1) != (zf2)) { + goto L_AES_GCM_decrypt_avx512_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_loadl_epi64((const __m128i*)WC_PR(rax, 0)); + x4 = _mm_insert_epi32(x4, (int)WC_L32(rax, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(x4, x5)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 128)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 144)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + zf3 = (word32)r10; + zf4 = 0xb; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 176)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + zf5 = (word32)r10; + zf6 = 0xd; + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 208)); + x5 = _mm_aesenc_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z1), + x7)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 224)); +L_AES_GCM_decrypt_avx512_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x7); + z1 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z1), + x7)); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1040), _mm512_castsi512_si128(z1)); + goto L_AES_GCM_decrypt_avx512_iv_done; +L_AES_GCM_decrypt_avx512_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf7 = (word32)r10; + zf8 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, _mm512_castsi512_si128(z9)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf9 = (word32)r10; + zf10 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, _mm512_castsi512_si128(z9)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, _mm512_castsi512_si128(z9)); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf11 = (word32)rdx; + zf12 = 0; + rcx = (word64)(0); + if ((zf11) == (zf12)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_decrypt_avx512_calc_iv_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rax, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x4, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x4, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x4)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x4 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x4, 31)); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z2)); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_16_loop; + } + rdx = (word32)((word32)rbx); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_done; + } +L_AES_GCM_decrypt_avx512_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_decrypt_avx512_calc_iv_loop: + r13 = (word32)((word32)WC_L8(rax, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x8); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x4, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x4, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x4)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x4 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x4, 31)); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z2)); +L_AES_GCM_decrypt_avx512_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x4, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x4, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x4)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x4 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x4, 31)); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_slli_epi32(x4, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x4 = _mm_or_si128(x4, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x4 = _mm_xor_si128(x4, _mm512_castsi512_si128(z2)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x8 = _mm_xor_si128(x8, x4); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf13 = (word32)r10; + zf14 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf15 = (word32)r10; + zf16 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_decrypt_avx512_calc_iv_2_aesenc_avx_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_calc_iv_2_aesenc_avx_last: + x8 = _mm_aesenclast_si128(x8, _mm512_castsi512_si128(z9)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1040), x8); +L_AES_GCM_decrypt_avx512_iv_done: + /* Additional authentication data */ + rdx = (word32)((word32)r11); + if (((word32)rdx) == (0)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_done; + } + rcx = (word32)(0); + if (((word32)rdx) < (0x80)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_ser; + } + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1088), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1104), x4); + z9 = _mm512_zextsi128_si512(_mm_srli_epi64(x5, 63)); + x8 = _mm_slli_epi64(x5, 1); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 8)); + x8 = _mm_or_si128(x8, _mm512_castsi512_si128(z9)); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + z2 = _mm512_zextsi128_si512(x6); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x0 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x0)); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z1 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm512_castsi512_si128(z1)); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z3 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm512_castsi512_si128(z3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z1))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z3), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z3))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z3), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z3), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + /* H ^ 9 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x7); + /* H ^ 10 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x7); + /* H ^ 11 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), x7); + /* H ^ 12 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), x7); + /* H ^ 13 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), x7); + /* H ^ 14 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), x7); + /* H ^ 15 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), x7); + /* H ^ 16 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), x7); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xffffff00); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_a256; + } +L_AES_GCM_decrypt_avx512_calc_aad_l256: + /* 256 bytes of AAD */ + rbx = (word64)(r12 + rcx); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rcx = (word32)((word32)rcx + 0x100); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_l256; + } +L_AES_GCM_decrypt_avx512_calc_aad_a256: + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_a128; + } + /* 128 bytes of AAD */ + rbx = (word64)(r12 + rcx); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rcx = (word32)((word32)rcx + 0x80); +L_AES_GCM_decrypt_avx512_calc_aad_a128: + r13 = (word32)((word32)r11); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rcx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_arem; + } + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_decrypt_avx512_calc_aad_rem: + /* 16 bytes of AAD */ + rbx = (word64)(r12 + rcx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rbx, 0)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x6)); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x6 = _mm512_castsi512_si128(z11); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_rem; + } +L_AES_GCM_decrypt_avx512_calc_aad_arem: + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1088)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1104)); + goto L_AES_GCM_decrypt_avx512_calc_aad_partial; +L_AES_GCM_decrypt_avx512_calc_aad_ser: + rdx = (word32)((word32)r11); + rdx = (word32)((word32)rdx & 0xfffffff0); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_partial; + } +L_AES_GCM_decrypt_avx512_calc_aad_16_loop: + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, rcx)); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x6, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x6)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x6 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x6 = _mm_xor_si128(x6, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x6, 31)); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x6 = _mm_or_si128(x6, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x6 = _mm_xor_si128(x6, _mm512_castsi512_si128(z2)); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_16_loop; + } +L_AES_GCM_decrypt_avx512_calc_aad_partial: + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_done; + } + rsp = (word64)(rsp - 16); + x8 = _mm_setzero_si128(); + rbx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); +L_AES_GCM_decrypt_avx512_calc_aad_loop: + r13 = (word32)((word32)WC_L8(r12, rcx)); + WC_S8(rsp, rbx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + rbx = (word32)((word32)rbx + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx512_calc_aad_loop; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + /* ghash_gfmul_avx */ + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z2 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z3 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x6, 0x11)); + x0 = _mm_clmulepi64_si128(x5, x6, 0); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x6)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x5)); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z2), 0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x0)); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z3))); + x7 = x0; + x6 = _mm512_castsi512_si128(z3); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 8)); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 8)); + x7 = _mm_xor_si128(x7, _mm512_castsi512_si128(z2)); + x6 = _mm_xor_si128(x6, _mm512_castsi512_si128(z1)); + x0 = _mm_srli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_srli_epi32(x6, 31)); + x7 = _mm_slli_epi32(x7, 1); + x6 = _mm_slli_epi32(x6, 1); + z2 = _mm512_zextsi128_si512(_mm_bsrli_si128(x0, 12)); + x0 = _mm_bslli_si128(x0, 4); + z1 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z1), 4)); + x6 = _mm_or_si128(x6, _mm512_castsi512_si128(z2)); + x7 = _mm_or_si128(x7, x0); + x6 = _mm_or_si128(x6, _mm512_castsi512_si128(z1)); + x0 = _mm_slli_epi32(x7, 31); + z1 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 30)); + z2 = _mm512_zextsi128_si512(_mm_slli_epi32(x7, 25)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z1)); + x0 = _mm_xor_si128(x0, _mm512_castsi512_si128(z2)); + z1 = _mm512_zextsi128_si512(x0); + z1 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z1), 4)); + x0 = _mm_bslli_si128(x0, 12); + x7 = _mm_xor_si128(x7, x0); + z2 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 1)); + z3 = _mm512_zextsi128_si512(_mm_srli_epi32(x7, 2)); + x0 = _mm_srli_epi32(x7, 7); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z3))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x0)); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z1))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), x7)); + x6 = _mm_xor_si128(x6, _mm512_castsi512_si128(z2)); +L_AES_GCM_decrypt_avx512_calc_aad_done: + /* Calculate counter and H */ + z9 = _mm512_zextsi128_si512(_mm_srli_epi64(x5, 63)); + x8 = _mm_slli_epi64(x5, 1); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 8)); + x8 = _mm_or_si128(x8, _mm512_castsi512_si128(z9)); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_one, 0))); + x5 = _mm_xor_si128(x5, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x4); + rbx = (word32)(0); + if (((word32)r9) < (0x80)) { + goto L_AES_GCM_decrypt_avx512_done_128; + } + z2 = _mm512_zextsi128_si512(x6); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x0 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x0)); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z1 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm512_castsi512_si128(z1)); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z3 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm512_castsi512_si128(z3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z1))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z3), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z3))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z3), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z3), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + if (((word32)r9) < (0x100)) { + goto L_AES_GCM_decrypt_avx512_no_ext2; + } + /* H ^ 9 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x7); + /* H ^ 10 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x7); + /* H ^ 11 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), x7); + /* H ^ 12 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), x7); + /* H ^ 13 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), x7); + /* H ^ 14 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), x7); + /* H ^ 15 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), x7); + /* H ^ 16 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), x7); + if (((word32)r9) < (0x200)) { + goto L_AES_GCM_decrypt_avx512_no_ext; + } + /* H ^ 17 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 128))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 256), x7); + /* H ^ 18 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 272), x7); + /* H ^ 19 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 144))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 288), x7); + /* H ^ 20 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 304), x7); + /* H ^ 21 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 160))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 320), x7); + /* H ^ 22 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 160)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 336), x7); + /* H ^ 23 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 160)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 176))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 352), x7); + /* H ^ 24 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 176)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 368), x7); + /* H ^ 25 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 176)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 192))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 384), x7); + /* H ^ 26 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 192)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 400), x7); + /* H ^ 27 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 192)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 208))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 416), x7); + /* H ^ 28 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 208)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 432), x7); + /* H ^ 29 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 208)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 224))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 448), x7); + /* H ^ 30 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 224)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 464), x7); + /* H ^ 31 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 224)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 240))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 480), x7); + /* H ^ 32 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 240)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 496), x7); +L_AES_GCM_decrypt_avx512_no_ext: +L_AES_GCM_decrypt_avx512_no_ext2: + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + z1 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + z3 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + if (((word32)r9) < (0x200)) { + goto L_AES_GCM_decrypt_avx512_no_windows; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffe00); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 448)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 384)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 320)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 256)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + _mm512_storeu_si512((void*)WC_PW(rsp, 512), z23); + _mm512_storeu_si512((void*)WC_PW(rsp, 576), z24); + _mm512_storeu_si512((void*)WC_PW(rsp, 640), z25); + _mm512_storeu_si512((void*)WC_PW(rsp, 704), z26); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + _mm512_storeu_si512((void*)WC_PW(rsp, 768), z23); + _mm512_storeu_si512((void*)WC_PW(rsp, 832), z24); + _mm512_storeu_si512((void*)WC_PW(rsp, 896), z25); + _mm512_storeu_si512((void*)WC_PW(rsp, 960), z26); + /* 512 bytes of input */ + r12 = (word32)(0); + rax = (word64)(rdi + rbx); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 512)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 576)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 640)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 704)); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 768)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 832)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 896)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 960)); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 320)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 384)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 448)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rbx = (word32)((word32)rbx + 0x200); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_last_aes; + } +L_AES_GCM_decrypt_avx512_win_loop: + rax = (word64)(rdi + rbx); + z21 = _mm512_setzero_si512(); + z21 = _mm512_inserti32x4(z21, x6, 0); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z31 = _mm512_loadu_si512((const void*)WC_PR(rax, 0)); + z31 = _mm512_shuffle_epi8(z31, z30); + z31 = _mm512_xor_si512(z31, z21); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0x11); + z27 = z23; + z28 = _mm512_xor_si512(z25, z24); + z29 = z26; + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z31 = _mm512_loadu_si512((const void*)WC_PR(rax, 64)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z31 = _mm512_loadu_si512((const void*)WC_PR(rax, 128)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z31 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf17 = (word32)r10; + zf18 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_decrypt_avx512_a_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf19 = (word32)r10; + zf20 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_decrypt_avx512_a_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_a_il_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r12 = (word32)((word32)r12 + 0x100); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z31 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z31 = _mm512_loadu_si512((const void*)WC_PR(rax, 320)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z31 = _mm512_loadu_si512((const void*)WC_PR(rax, 384)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z31 = _mm512_loadu_si512((const void*)WC_PR(rax, 448)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf21 = (word32)r10; + zf22 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_decrypt_avx512_b_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf23 = (word32)r10; + zf24 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_decrypt_avx512_b_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_b_il_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r12 = (word32)((word32)r12 + 0x100); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z23 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z23, 0x96); + z23 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z23, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rbx = (word32)((word32)rbx + 0x200); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_win_loop; + } +L_AES_GCM_decrypt_avx512_last_aes: + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf25 = (word32)r10; + zf26 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_decrypt_avx512_l1_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf27 = (word32)r10; + zf28 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_decrypt_avx512_l1_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_l1_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r12 = (word32)((word32)r12 + 0x100); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf29 = (word32)r10; + zf30 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_decrypt_avx512_l2_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf31 = (word32)r10; + zf32 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_GCM_decrypt_avx512_l2_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_l2_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r12 = (word32)((word32)r12 + 0x100); +L_AES_GCM_decrypt_avx512_no_windows: + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xffffff00); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_after_256; + } + /* 256 bytes of input */ + rax = (word64)(rdi + rbx); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf33 = (word32)r10; + zf34 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_GCM_decrypt_avx512_t_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf35 = (word32)r10; + zf36 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf35) < (sword32)(zf36)) { + goto L_AES_GCM_decrypt_avx512_t_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_t_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + rbx = (word32)((word32)rbx + 0x100); +L_AES_GCM_decrypt_avx512_after_256: + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_after_128; + } + /* 128 bytes of input */ + rax = (word64)(rdi + rbx); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rax, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 16))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 32))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 48))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 64))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 80))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 96))); + z1 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 112))); + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 128))); + z3 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 144))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_eight, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + zf37 = (word32)r10; + zf38 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf37) < (sword32)(zf38)) { + goto L_AES_GCM_decrypt_avx512_8_avx512_ctr8_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + zf39 = (word32)r10; + zf40 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf39) < (sword32)(zf40)) { + goto L_AES_GCM_decrypt_avx512_8_avx512_ctr8_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_8_avx512_ctr8_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + rcx = (word64)(rdi + rbx); + rdx = (word64)(rsi + rbx); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + rbx = (word32)((word32)rbx + 0x80); +L_AES_GCM_decrypt_avx512_after_128: + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_decrypt_avx512_done_128: + rdx = (word32)((word32)r9); + if (((word32)rbx) >= ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx512_done_dec; + } + r13 = (word32)((word32)r9); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)rbx) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_last_block_done; + } +L_AES_GCM_decrypt_avx512_last_block_start: + z13 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + rbx))); + x0 = x5; + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi8(_mm512_castsi512_si128(z13), + _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_bswap_mask, + 0)))); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x6)); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 1024))); + x8 = _mm_shuffle_epi8(_mm512_castsi512_si128(z9), _mm_loadu_si128(( + const __m128i*)WC_PR(L_avx512_aes_gcm_bswap_epi64, 0))); + z9 = _mm512_zextsi128_si512(_mm_add_epi32(_mm512_castsi512_si128(z9), + _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 1024), _mm512_castsi512_si128(z9)); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + z10 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 1)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + x0, 0x11)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 8)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z10), + 8)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z12))); + z3 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z10))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z2), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z2), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z10), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z10), _mm512_castsi512_si128(z3)); + zf41 = (word32)r10; + zf42 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf41) < (sword32)(zf42)) { + goto L_AES_GCM_decrypt_avx512_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf43 = (word32)r10; + zf44 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf43) < (sword32)(zf44)) { + goto L_AES_GCM_decrypt_avx512_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, _mm512_castsi512_si128(z9)); + x0 = _mm512_castsi512_si128(z13); + x8 = _mm_xor_si128(x8, x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, rbx), x8); + rbx = (word32)((word32)rbx + 0x10); + if (((word32)rbx) < ((word32)r13)) { + goto L_AES_GCM_decrypt_avx512_last_block_start; + } +L_AES_GCM_decrypt_avx512_last_block_done: + rcx = (word32)((word32)r9); + rdx = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0xf); + if (((word32)rcx) == (0)) { + goto L_AES_GCM_decrypt_avx512_aesenc_last15_dec_avx_done; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1024)); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + x4 = _mm_xor_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 16))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 32))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 48))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 64))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 80))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 96))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 112))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 128))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 144))); + zf45 = (word32)r10; + zf46 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 160))); + if ((sword32)(zf45) < (sword32)(zf46)) { + goto L_AES_GCM_decrypt_avx512_aesenc_last15_dec_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, _mm512_castsi512_si128(z9)); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 176))); + zf47 = (word32)r10; + zf48 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 192))); + if ((sword32)(zf47) < (sword32)(zf48)) { + goto L_AES_GCM_decrypt_avx512_aesenc_last15_dec_avx_aesenc_avx_last; + } + x4 = _mm_aesenc_si128(x4, _mm512_castsi512_si128(z9)); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(r15, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r15, + 224))); +L_AES_GCM_decrypt_avx512_aesenc_last15_dec_avx_aesenc_avx_last: + x4 = _mm_aesenclast_si128(x4, _mm512_castsi512_si128(z9)); + rsp = (word64)(rsp - 32); + rcx = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + x0 = _mm_setzero_si128(); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); +L_AES_GCM_decrypt_avx512_aesenc_last15_dec_avx_loop: + r13 = (word32)((word32)WC_L8(rdi, rbx)); + WC_S8(rsp, rcx + 16) = (byte)((byte)r13); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsp, + rcx)) & 0xff); + WC_S8(rsi, rbx) = (byte)((byte)r13); + rbx = (word32)((word32)rbx + 1); + rcx = (word32)((word32)rcx + 1); + if (((word32)rbx) < ((word32)rdx)) { + goto L_AES_GCM_decrypt_avx512_aesenc_last15_dec_avx_loop; + } + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 16)); + rsp = (word64)(rsp + 32); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x4); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x6)); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x6 = _mm512_castsi512_si128(z11); +L_AES_GCM_decrypt_avx512_aesenc_last15_dec_avx_done: +L_AES_GCM_decrypt_avx512_done_dec: + rdx = (word32)((word32)r9); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + z1 = _mm512_zextsi128_si512(_mm_cvtsi64_si128((long long)rcx)); + x0 = _mm_unpacklo_epi64(x0, _mm512_castsi512_si128(z1)); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x6)); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x6 = _mm512_castsi512_si128(z11); + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 1040)); + x0 = _mm_xor_si128(x0, x6); + if (((word32)r14) == (0x10)) { + goto L_AES_GCM_decrypt_avx512_cmp_tag_16; + } + rsp = (word64)(rsp - 16); + rcx = (word64)(0); + rbx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_avx512_cmp_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(r8, + rcx)) & 0xff); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)((byte)rbx | ( + byte)r13) & 0xff); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)r14)) { + goto L_AES_GCM_decrypt_avx512_cmp_tag_loop; + } + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)(((byte)rbx) == (0)) & 0xff); + rsp = (word64)(rsp + 16); + rcx = (word64)(0); + goto L_AES_GCM_decrypt_avx512_cmp_tag_done; +L_AES_GCM_decrypt_avx512_cmp_tag_16: + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x0 = _mm_cmpeq_epi8(x0, _mm512_castsi512_si128(z1)); + rdx = (word32)((word32)_mm_movemask_epi8(x0)); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + rbx = (word32)(0); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)(((word32)rdx) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_avx512_cmp_tag_done: + WC_S32(rbp, 0) = (word32)((word32)rbx); +} + +#ifdef WOLFSSL_AESGCM_STREAM +WC_X64I_TARGET("aes,pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_init_avx512(const unsigned char* key, int nr, + const unsigned char* ivec, unsigned int ibytes, unsigned char* h, + unsigned char* counter, unsigned char* initCtr) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, rsp, rdx, rcx = 0, r13 = 0, + r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)ivec; + r11 = (word64)(word32)ibytes; + r8 = (word64)(size_t)h; + r9 = (word64)(size_t)counter; + rax = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_setzero_si128(); + rdx = (word32)((word32)r11); + if (((word32)rdx) != (0xc)) { + goto L_AES_GCM_init_avx512_iv_not_12; + } + /* # Calculate values when IV is 12 bytes */ + /* Set counter based on IV */ + rcx = (word32)(0x1000000); + x4 = _mm_loadl_epi64((const __m128i*)WC_PR(r10, 0)); + x4 = _mm_insert_epi32(x4, (int)WC_L32(r10, 8), 2); + x4 = _mm_insert_epi32(x4, (int)(word32)rcx, 3); + /* H = Encrypt X(=0) and T = Encrypt counter */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_xor_si128(x4, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + zf1 = (word32)rsi; + zf2 = 0xb; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_init_avx512_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + zf3 = (word32)rsi; + zf4 = 0xd; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_init_avx512_calc_iv_12_last; + } + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208)); + x5 = _mm_aesenc_si128(x5, x6); + x1 = _mm_aesenc_si128(x1, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_avx512_calc_iv_12_last: + x5 = _mm_aesenclast_si128(x5, x6); + x1 = _mm_aesenclast_si128(x1, x6); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x15 = x1; + goto L_AES_GCM_init_avx512_iv_done; +L_AES_GCM_init_avx512_iv_not_12: + /* Calculate values when IV is not 12 bytes */ + /* H = Encrypt X(=0) */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf5 = (word32)rsi; + zf6 = 0xb; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_init_avx512_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x8); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf7 = (word32)rsi; + zf8 = 0xd; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_init_avx512_calc_iv_1_aesenc_avx_last; + } + x5 = _mm_aesenc_si128(x5, x8); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_avx512_calc_iv_1_aesenc_avx_last: + x5 = _mm_aesenclast_si128(x5, x8); + x5 = _mm_shuffle_epi8(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + /* Calc counter */ + /* Initialization vector */ + zf9 = (word32)rdx; + zf10 = 0; + rcx = (word64)(0); + if ((zf9) == (zf10)) { + goto L_AES_GCM_init_avx512_calc_iv_done; + } + if (((word32)rdx) < (0x10)) { + goto L_AES_GCM_init_avx512_calc_iv_lt16; + } + rdx = (word32)((word32)rdx & 0xfffffff0); +L_AES_GCM_init_avx512_calc_iv_16_loop: + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, rcx)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + rcx = (word32)((word32)rcx + 0x10); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_avx512_calc_iv_16_loop; + } + rdx = (word32)((word32)r11); + if (((word32)rcx) == ((word32)rdx)) { + goto L_AES_GCM_init_avx512_calc_iv_done; + } +L_AES_GCM_init_avx512_calc_iv_lt16: + rsp = (word64)(rsp - 16); + x7 = _mm_setzero_si128(); + r13 = (word32)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x7); +L_AES_GCM_init_avx512_calc_iv_loop: + r12 = (word32)((word32)WC_L8(r10, rcx)); + WC_S8(rsp, r13) = (byte)((byte)r12); + rcx = (word32)((word32)rcx + 1); + r13 = (word32)((word32)r13 + 1); + if (((word32)rcx) < ((word32)rdx)) { + goto L_AES_GCM_init_avx512_calc_iv_loop; + } + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + rsp = (word64)(rsp + 16); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); +L_AES_GCM_init_avx512_calc_iv_done: + /* T = Encrypt counter */ + x0 = _mm_setzero_si128(); + rdx = (word32)((word32)rdx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + /* Encrypt counter */ + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x7 = _mm_xor_si128(x7, x4); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf11 = (word32)rsi; + zf12 = 0xb; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_init_avx512_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x8); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf13 = (word32)rsi; + zf14 = 0xd; + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_init_avx512_calc_iv_2_aesenc_avx_last; + } + x7 = _mm_aesenc_si128(x7, x8); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_init_avx512_calc_iv_2_aesenc_avx_last: + x7 = _mm_aesenclast_si128(x7, x8); + x15 = x7; +L_AES_GCM_init_avx512_iv_done: + _mm_storeu_si128((__m128i*)WC_PW(rax, 0), x15); + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x4); +} + +WC_X64I_TARGET("pclmul,vpclmulqdq,avx,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_GCM_aad_update_avx512(const unsigned char* addt, + unsigned int abytes, unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, r8, r9, rsp, rdx, rax, rcx = 0; + __m128i x0, x1, x2, x3, x4 = _mm_setzero_si128(), + x5 = _mm_setzero_si128(), x6, x7, x8, x9, x10, x11, x12, x13; + __m512i z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(), z22, + z23, z24, z25, z26, z27, z28 = _mm512_setzero_si512(), + z29 = _mm512_setzero_si512(), z30 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[32]; + + rdi = (word64)(size_t)addt; + rsi = (word64)(word32)abytes; + r8 = (word64)(size_t)tag; + r9 = (word64)(size_t)h; + + rsp = (word64)(size_t)stk + 256; + rsp = (word64)(rsp - 256); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + rdx = (word32)(0); + x10 = _mm_srli_epi64(x7, 63); + x9 = _mm_slli_epi64(x7, 1); + x10 = _mm_bslli_si128(x10, 8); + x9 = _mm_or_si128(x9, x10); + x7 = _mm_shuffle_epi32(x7, 0xff); + x7 = _mm_srai_epi32(x7, 31); + x7 = _mm_and_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + x7 = _mm_xor_si128(x7, x9); + x2 = x6; + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x7); + /* H ^ 2 */ + x9 = _mm_clmulepi64_si128(x7, x7, 0); + x12 = _mm_clmulepi64_si128(x7, x7, 0x11); + x10 = _mm_setzero_si128(); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x0 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + x10 = _mm_shuffle_epi32(x7, 0x4e); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_shuffle_epi32(x0, 0x4e); + x11 = _mm_xor_si128(x11, x0); + x9 = _mm_clmulepi64_si128(x0, x7, 0); + x12 = _mm_clmulepi64_si128(x0, x7, 0x11); + x10 = _mm_clmulepi64_si128(x10, x11, 0); + x10 = _mm_ternarylogic_epi64(x10, x12, x9, 0x96); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x1 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x1); + /* H ^ 4 */ + x9 = _mm_clmulepi64_si128(x0, x0, 0); + x12 = _mm_clmulepi64_si128(x0, x0, 0x11); + x10 = _mm_setzero_si128(); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x3 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x3); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + x10 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_shuffle_epi32(x1, 0x4e); + x11 = _mm_xor_si128(x11, x1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x10 = _mm_clmulepi64_si128(x10, x11, 0); + x10 = _mm_ternarylogic_epi64(x10, x12, x9, 0x96); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + /* H ^ 6 */ + x9 = _mm_clmulepi64_si128(x1, x1, 0); + x12 = _mm_clmulepi64_si128(x1, x1, 0x11); + x10 = _mm_setzero_si128(); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x8); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + x10 = _mm_shuffle_epi32(x1, 0x4e); + x10 = _mm_xor_si128(x10, x1); + x11 = _mm_shuffle_epi32(x3, 0x4e); + x11 = _mm_xor_si128(x11, x3); + x9 = _mm_clmulepi64_si128(x3, x1, 0); + x12 = _mm_clmulepi64_si128(x3, x1, 0x11); + x10 = _mm_clmulepi64_si128(x10, x11, 0); + x10 = _mm_ternarylogic_epi64(x10, x12, x9, 0x96); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + /* H ^ 8 */ + x9 = _mm_clmulepi64_si128(x3, x3, 0); + x12 = _mm_clmulepi64_si128(x3, x3, 0x11); + x10 = _mm_setzero_si128(); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x8); + /* H ^ 9 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + /* ghash_gfmul_red_avx */ + x10 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_shuffle_epi32(x1, 0x4e); + x11 = _mm_xor_si128(x11, x1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x10 = _mm_clmulepi64_si128(x10, x11, 0); + x10 = _mm_ternarylogic_epi64(x10, x12, x9, 0x96); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x8); + /* H ^ 10 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x9 = _mm_clmulepi64_si128(x0, x0, 0); + x12 = _mm_clmulepi64_si128(x0, x0, 0x11); + x10 = _mm_setzero_si128(); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x8); + /* H ^ 11 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + /* ghash_gfmul_red_avx */ + x10 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_shuffle_epi32(x1, 0x4e); + x11 = _mm_xor_si128(x11, x1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x10 = _mm_clmulepi64_si128(x10, x11, 0); + x10 = _mm_ternarylogic_epi64(x10, x12, x9, 0x96); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), x8); + /* H ^ 12 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x9 = _mm_clmulepi64_si128(x0, x0, 0); + x12 = _mm_clmulepi64_si128(x0, x0, 0x11); + x10 = _mm_setzero_si128(); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), x8); + /* H ^ 13 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + /* ghash_gfmul_red_avx */ + x10 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_shuffle_epi32(x1, 0x4e); + x11 = _mm_xor_si128(x11, x1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x10 = _mm_clmulepi64_si128(x10, x11, 0); + x10 = _mm_ternarylogic_epi64(x10, x12, x9, 0x96); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), x8); + /* H ^ 14 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x9 = _mm_clmulepi64_si128(x0, x0, 0); + x12 = _mm_clmulepi64_si128(x0, x0, 0x11); + x10 = _mm_setzero_si128(); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), x8); + /* H ^ 15 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + /* ghash_gfmul_red_avx */ + x10 = _mm_shuffle_epi32(x0, 0x4e); + x10 = _mm_xor_si128(x10, x0); + x11 = _mm_shuffle_epi32(x1, 0x4e); + x11 = _mm_xor_si128(x11, x1); + x9 = _mm_clmulepi64_si128(x1, x0, 0); + x12 = _mm_clmulepi64_si128(x1, x0, 0x11); + x10 = _mm_clmulepi64_si128(x10, x11, 0); + x10 = _mm_ternarylogic_epi64(x10, x12, x9, 0x96); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), x8); + /* H ^ 16 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x9 = _mm_clmulepi64_si128(x0, x0, 0); + x12 = _mm_clmulepi64_si128(x0, x0, 0x11); + x10 = _mm_setzero_si128(); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x8 = x12; + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), x8); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + z23 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + z27 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z27 = _mm512_shuffle_i64x2(z27, z27, 0x1b); + rax = (word32)((word32)rsi); + rax = (word32)((word32)rax & 0xffffff00); + if (((word32)rdx) >= ((word32)rax)) { + goto L_AES_GCM_aad_update_avx512_after_256; + } +L_AES_GCM_aad_update_avx512_loop_256: + /* 256 bytes of input */ + rcx = (word64)(rdi + rdx); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z21 = _mm512_shuffle_epi8(z21, z22); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z28 = z16; + z29 = _mm512_xor_si512(z18, z17); + z30 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z21 = _mm512_shuffle_epi8(z21, z22); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z28 = _mm512_xor_si512(z28, z16); + z29 = _mm512_ternarylogic_epi64(z29, z18, z17, 0x96); + z30 = _mm512_xor_si512(z30, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z21 = _mm512_shuffle_epi8(z21, z22); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z28 = _mm512_xor_si512(z28, z16); + z29 = _mm512_ternarylogic_epi64(z29, z18, z17, 0x96); + z30 = _mm512_xor_si512(z30, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z21 = _mm512_shuffle_epi8(z21, z22); + z16 = _mm512_clmulepi64_epi128(z21, z27, 0); + z17 = _mm512_clmulepi64_epi128(z21, z27, 1); + z18 = _mm512_clmulepi64_epi128(z21, z27, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z27, 0x11); + z28 = _mm512_xor_si512(z28, z16); + z29 = _mm512_ternarylogic_epi64(z29, z18, z17, 0x96); + z30 = _mm512_xor_si512(z30, z19); + z21 = _mm512_clmulepi64_epi128(z23, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z23, z29, 1); + z29 = _mm512_shuffle_epi32(z29, 0x4e); + z30 = _mm512_ternarylogic_epi64(z30, z29, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z30, 1); + x4 = _mm512_extracti32x4_epi32(z30, 2); + x5 = _mm512_extracti32x4_epi32(z30, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z30), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rdx = (word32)((word32)rdx + 0x100); + if (((word32)rdx) < ((word32)rax)) { + goto L_AES_GCM_aad_update_avx512_loop_256; + } +L_AES_GCM_aad_update_avx512_after_256: + rax = (word32)((word32)rsi); + rax = (word32)((word32)rax & 0xffffff80); + if (((word32)rdx) >= ((word32)rax)) { + goto L_AES_GCM_aad_update_avx512_after_128; + } + /* 128 bytes of input */ + rcx = (word64)(rdi + rdx); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z21 = _mm512_shuffle_epi8(z21, z22); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z28 = z16; + z29 = _mm512_xor_si512(z18, z17); + z30 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z21 = _mm512_shuffle_epi8(z21, z22); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z28 = _mm512_xor_si512(z28, z16); + z29 = _mm512_ternarylogic_epi64(z29, z18, z17, 0x96); + z30 = _mm512_xor_si512(z30, z19); + z23 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z21 = _mm512_clmulepi64_epi128(z23, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z23, z29, 1); + z29 = _mm512_shuffle_epi32(z29, 0x4e); + z30 = _mm512_ternarylogic_epi64(z30, z29, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z30, 1); + x4 = _mm512_extracti32x4_epi32(z30, 2); + x5 = _mm512_extracti32x4_epi32(z30, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z30), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + rdx = (word32)((word32)rdx + 0x80); +L_AES_GCM_aad_update_avx512_after_128: + rax = (word32)((word32)rsi); + rax = (word32)((word32)rax & 0xfffffff0); + if (((word32)rdx) >= ((word32)rax)) { + goto L_AES_GCM_aad_update_avx512_done; + } + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_aad_update_avx512_tail: + /* 16 bytes of input */ + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, rdx)); + x9 = _mm_shuffle_epi8(x9, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x9); + /* ghash_gfmul_red_avx */ + x10 = _mm_shuffle_epi32(x7, 0x4e); + x10 = _mm_xor_si128(x10, x7); + x11 = _mm_shuffle_epi32(x6, 0x4e); + x11 = _mm_xor_si128(x11, x6); + x9 = _mm_clmulepi64_si128(x6, x7, 0); + x12 = _mm_clmulepi64_si128(x6, x7, 0x11); + x10 = _mm_clmulepi64_si128(x10, x11, 0); + x10 = _mm_ternarylogic_epi64(x10, x12, x9, 0x96); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x13 = _mm_clmulepi64_si128(x11, x9, 1); + x9 = _mm_shuffle_epi32(x9, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x9, x13, 0x96); + x13 = _mm_clmulepi64_si128(x11, x10, 1); + x10 = _mm_shuffle_epi32(x10, 0x4e); + x12 = _mm_ternarylogic_epi64(x12, x10, x13, 0x96); + x6 = x12; + rdx = (word32)((word32)rdx + 0x10); + if (((word32)rdx) < ((word32)rax)) { + goto L_AES_GCM_aad_update_avx512_tail; + } +L_AES_GCM_aad_update_avx512_done: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x6); + (void)x2; +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_GCM_encrypt_block_avx512(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8; + __m128i x0, x1; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(size_t)counter; + + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x0 = _mm_shuffle_epi8(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_one, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x1); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf1 = (word32)rsi; + zf2 = 0xb; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_block_avx512_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf3 = (word32)rsi; + zf4 = 0xd; + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_block_avx512_aesenc_block_last; + } + x0 = _mm_aesenc_si128(x0, x1); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 224)); +L_AES_GCM_encrypt_block_avx512_aesenc_block_last: + x0 = _mm_aesenclast_si128(x0, x1); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); +} + +WC_X64I_TARGET("pclmul,avx") +WOLFSSL_LOCAL void AES_GCM_ghash_block_avx512(const unsigned char* addt, + unsigned char* tag, unsigned char* h) +{ + word64 rdi, rsi, rdx; + __m128i x0, x1, x2, x3, x4, x5, x6, x7; + + rdi = (word64)(size_t)addt; + rsi = (word64)(size_t)tag; + rdx = (word64)(size_t)h; + + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x4 = _mm_xor_si128(x4, x7); + /* ghash_gfmul_avx */ + x1 = _mm_shuffle_epi32(x4, 0x4e); + x2 = _mm_shuffle_epi32(x5, 0x4e); + x3 = _mm_clmulepi64_si128(x5, x4, 0x11); + x0 = _mm_clmulepi64_si128(x5, x4, 0); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x5); + x1 = _mm_clmulepi64_si128(x1, x2, 0); + x1 = _mm_xor_si128(x1, x0); + x1 = _mm_xor_si128(x1, x3); + x6 = x0; + x4 = x3; + x2 = _mm_bslli_si128(x1, 8); + x1 = _mm_bsrli_si128(x1, 8); + x6 = _mm_xor_si128(x6, x2); + x4 = _mm_xor_si128(x4, x1); + x0 = _mm_srli_epi32(x6, 31); + x1 = _mm_srli_epi32(x4, 31); + x6 = _mm_slli_epi32(x6, 1); + x4 = _mm_slli_epi32(x4, 1); + x2 = _mm_bsrli_si128(x0, 12); + x0 = _mm_bslli_si128(x0, 4); + x1 = _mm_bslli_si128(x1, 4); + x4 = _mm_or_si128(x4, x2); + x6 = _mm_or_si128(x6, x0); + x4 = _mm_or_si128(x4, x1); + x0 = _mm_slli_epi32(x6, 31); + x1 = _mm_slli_epi32(x6, 30); + x2 = _mm_slli_epi32(x6, 25); + x0 = _mm_xor_si128(x0, x1); + x0 = _mm_xor_si128(x0, x2); + x1 = x0; + x1 = _mm_bsrli_si128(x1, 4); + x0 = _mm_bslli_si128(x0, 12); + x6 = _mm_xor_si128(x6, x0); + x2 = _mm_srli_epi32(x6, 1); + x3 = _mm_srli_epi32(x6, 2); + x0 = _mm_srli_epi32(x6, 7); + x2 = _mm_xor_si128(x2, x3); + x2 = _mm_xor_si128(x2, x0); + x2 = _mm_xor_si128(x2, x1); + x2 = _mm_xor_si128(x2, x6); + x4 = _mm_xor_si128(x4, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x4); +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_GCM_encrypt_update_avx512(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, rbp = 0, r15 = 0, + rcx = 0, rdx = 0, rbx = 0, r13 = 0; + __m128i x0 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), x5, x6, + x7 = _mm_setzero_si128(), x8; + __m512i z1 = _mm512_setzero_si512(), z2 = _mm512_setzero_si512(), + z3 = _mm512_setzero_si512(), z9, z10 = _mm512_setzero_si512(), + z11 = _mm512_setzero_si512(), z12 = _mm512_setzero_si512(), + z13 = _mm512_setzero_si512(), z14 = _mm512_setzero_si512(), + z15 = _mm512_setzero_si512(), z16 = _mm512_setzero_si512(), + z17 = _mm512_setzero_si512(), z18 = _mm512_setzero_si512(), + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(), + z23 = _mm512_setzero_si512(), z24 = _mm512_setzero_si512(), + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(), + z27 = _mm512_setzero_si512(), z28 = _mm512_setzero_si512(), + z29 = _mm512_setzero_si512(), z30 = _mm512_setzero_si512(), + z31 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[132]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + word32 zf35; + word32 zf36; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 1056; + rsp = (word64)(rsp - 1040); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + z9 = _mm512_zextsi128_si512(_mm_srli_epi64(x5, 63)); + x8 = _mm_slli_epi64(x5, 1); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 8)); + x8 = _mm_or_si128(x8, _mm512_castsi512_si128(z9)); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + r14 = (word32)(0); + if (((word32)r8) < (0x80)) { + goto L_AES_GCM_encrypt_update_avx512_done_128; + } + z2 = _mm512_zextsi128_si512(x6); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x0 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x0)); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z1 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm512_castsi512_si128(z1)); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z3 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm512_castsi512_si128(z3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z1))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z3), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z3))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z3), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z3), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + if (((word32)r8) < (0x100)) { + goto L_AES_GCM_encrypt_update_avx512_no_ext2; + } + /* H ^ 9 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x7); + /* H ^ 10 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x7); + /* H ^ 11 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), x7); + /* H ^ 12 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), x7); + /* H ^ 13 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), x7); + /* H ^ 14 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), x7); + /* H ^ 15 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), x7); + /* H ^ 16 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), x7); + if (((word32)r8) < (0x200)) { + goto L_AES_GCM_encrypt_update_avx512_no_ext; + } + /* H ^ 17 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 128))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 256), x7); + /* H ^ 18 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 272), x7); + /* H ^ 19 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 144))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 288), x7); + /* H ^ 20 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 304), x7); + /* H ^ 21 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 160))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 320), x7); + /* H ^ 22 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 160)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 336), x7); + /* H ^ 23 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 160)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 176))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 352), x7); + /* H ^ 24 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 176)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 368), x7); + /* H ^ 25 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 176)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 192))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 384), x7); + /* H ^ 26 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 192)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 400), x7); + /* H ^ 27 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 192)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 208))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 416), x7); + /* H ^ 28 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 208)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 432), x7); + /* H ^ 29 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 208)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 224))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 448), x7); + /* H ^ 30 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 224)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 464), x7); + /* H ^ 31 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 224)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 240))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 480), x7); + /* H ^ 32 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 240)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 496), x7); +L_AES_GCM_encrypt_update_avx512_no_ext: +L_AES_GCM_encrypt_update_avx512_no_ext2: + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + z1 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + z3 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + if (((word32)r8) < (0x200)) { + goto L_AES_GCM_encrypt_update_avx512_no_windows; + } + rbp = (word32)((word32)r8); + rbp = (word32)((word32)rbp & 0xfffffe00); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 448)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 384)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 320)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 256)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + _mm512_storeu_si512((void*)WC_PW(rsp, 512), z23); + _mm512_storeu_si512((void*)WC_PW(rsp, 576), z24); + _mm512_storeu_si512((void*)WC_PW(rsp, 640), z25); + _mm512_storeu_si512((void*)WC_PW(rsp, 704), z26); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + _mm512_storeu_si512((void*)WC_PW(rsp, 768), z23); + _mm512_storeu_si512((void*)WC_PW(rsp, 832), z24); + _mm512_storeu_si512((void*)WC_PW(rsp, 896), z25); + _mm512_storeu_si512((void*)WC_PW(rsp, 960), z26); + /* 512 bytes of input */ + r15 = (word64)(r10 + r14); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf1 = (word32)rsi; + zf2 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_encrypt_update_avx512_p1_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf3 = (word32)rsi; + zf4 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_encrypt_update_avx512_p1_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_avx512_p1_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r14 = (word32)((word32)r14 + 0x100); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf5 = (word32)rsi; + zf6 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_encrypt_update_avx512_p2_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf7 = (word32)rsi; + zf8 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_encrypt_update_avx512_p2_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_avx512_p2_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r14 = (word32)((word32)r14 + 0x100); + if (((word32)r14) >= ((word32)rbp)) { + goto L_AES_GCM_encrypt_update_avx512_last_win; + } +L_AES_GCM_encrypt_update_avx512_win_loop: + rbx = (word64)(r10 + r14); + z21 = _mm512_setzero_si512(); + z21 = _mm512_inserti32x4(z21, x6, 0); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z31 = _mm512_loadu_si512((const void*)WC_PR(r15, 0)); + z31 = _mm512_shuffle_epi8(z31, z30); + z31 = _mm512_xor_si512(z31, z21); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0x11); + z27 = z23; + z28 = _mm512_xor_si512(z25, z24); + z29 = z26; + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z31 = _mm512_loadu_si512((const void*)WC_PR(r15, 64)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z31 = _mm512_loadu_si512((const void*)WC_PR(r15, 128)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z31 = _mm512_loadu_si512((const void*)WC_PR(r15, 192)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf9 = (word32)rsi; + zf10 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_encrypt_update_avx512_a_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf11 = (word32)rsi; + zf12 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_encrypt_update_avx512_a_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_avx512_a_il_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r14 = (word32)((word32)r14 + 0x100); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z31 = _mm512_loadu_si512((const void*)WC_PR(r15, 256)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z31 = _mm512_loadu_si512((const void*)WC_PR(r15, 320)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z31 = _mm512_loadu_si512((const void*)WC_PR(r15, 384)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z31 = _mm512_loadu_si512((const void*)WC_PR(r15, 448)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf13 = (word32)rsi; + zf14 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_encrypt_update_avx512_b_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf15 = (word32)rsi; + zf16 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_encrypt_update_avx512_b_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_avx512_b_il_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r14 = (word32)((word32)r14 + 0x100); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z23 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z23, 0x96); + z23 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z23, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + r15 = (word64)(rbx); + if (((word32)r14) < ((word32)rbp)) { + goto L_AES_GCM_encrypt_update_avx512_win_loop; + } +L_AES_GCM_encrypt_update_avx512_last_win: + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 512)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 576)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 640)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 704)); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 768)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 832)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 896)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 960)); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 256)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 320)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 384)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 448)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); +L_AES_GCM_encrypt_update_avx512_no_windows: + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xffffff00); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx512_after_256; + } + /* 256 bytes of input */ + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf17 = (word32)rsi; + zf18 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_encrypt_update_avx512_pro_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf19 = (word32)rsi; + zf20 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_encrypt_update_avx512_pro_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_avx512_pro_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r15 = (word64)(rdx); + r14 = (word32)((word32)r14 + 0x100); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx512_last_ghash; + } +L_AES_GCM_encrypt_update_avx512_ghash_128: + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf21 = (word32)rsi; + zf22 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_encrypt_update_avx512_pip_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf23 = (word32)rsi; + zf24 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_encrypt_update_avx512_pip_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_avx512_pip_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + r15 = (word64)(rdx); + r14 = (word32)((word32)r14 + 0x100); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx512_ghash_128; + } +L_AES_GCM_encrypt_update_avx512_last_ghash: + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); +L_AES_GCM_encrypt_update_avx512_after_256: + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx512_after_128; + } + /* 128 bytes of input */ + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + z1 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + z3 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + r15 = (word64)(r10 + r14); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_eight, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + zf25 = (word32)rsi; + zf26 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_encrypt_update_avx512_8_avx512_ctr8_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + zf27 = (word32)rsi; + zf28 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_encrypt_update_avx512_8_avx512_ctr8_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_avx512_8_avx512_ctr8_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + r14 = (word32)((word32)r14 + 0x80); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(r15, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); +L_AES_GCM_encrypt_update_avx512_after_128: + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_encrypt_update_avx512_done_128: + rdx = (word32)((word32)r8); + if (((word32)r14) >= ((word32)rdx)) { + goto L_AES_GCM_encrypt_update_avx512_done_enc; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx512_last_block_done; + } + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x8 = _mm_shuffle_epi8(_mm512_castsi512_si128(z9), _mm_loadu_si128(( + const __m128i*)WC_PR(L_avx512_aes_gcm_bswap_epi64, 0))); + z9 = _mm512_zextsi128_si512(_mm_add_epi32(_mm512_castsi512_si128(z9), + _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm512_castsi512_si128(z9)); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + zf29 = (word32)rsi; + zf30 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_GCM_encrypt_update_avx512_aesenc_block_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf31 = (word32)rsi; + zf32 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_GCM_encrypt_update_avx512_aesenc_block_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_avx512_aesenc_block_last: + x8 = _mm_aesenclast_si128(x8, _mm512_castsi512_si128(z9)); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r11, + r14))); + x8 = _mm_xor_si128(x8, _mm512_castsi512_si128(z9)); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x6 = _mm_xor_si128(x6, x8); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx512_last_block_ghash; + } +L_AES_GCM_encrypt_update_avx512_last_block_start: + z13 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r11, + r14))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x8 = _mm_shuffle_epi8(_mm512_castsi512_si128(z9), _mm_loadu_si128(( + const __m128i*)WC_PR(L_avx512_aes_gcm_bswap_epi64, 0))); + z9 = _mm512_zextsi128_si512(_mm_add_epi32(_mm512_castsi512_si128(z9), + _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm512_castsi512_si128(z9)); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + z10 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 1)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 8)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z10), + 8)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z12))); + z3 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z10))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z2), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z2), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z10), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z10), _mm512_castsi512_si128(z3)); + zf33 = (word32)rsi; + zf34 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_GCM_encrypt_update_avx512_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf35 = (word32)rsi; + zf36 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf35) < (sword32)(zf36)) { + goto L_AES_GCM_encrypt_update_avx512_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_encrypt_update_avx512_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, _mm512_castsi512_si128(z9)); + x0 = _mm512_castsi512_si128(z13); + x8 = _mm_xor_si128(x8, x0); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), x8); + x8 = _mm_shuffle_epi8(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + r14 = (word32)((word32)r14 + 0x10); + x6 = _mm_xor_si128(x6, x8); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_encrypt_update_avx512_last_block_start; + } +L_AES_GCM_encrypt_update_avx512_last_block_ghash: + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x6, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x6)); + x8 = _mm_clmulepi64_si128(x6, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x6, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x6 = _mm512_castsi512_si128(z11); +L_AES_GCM_encrypt_update_avx512_last_block_done: +L_AES_GCM_encrypt_update_avx512_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x6); +} + +WC_X64I_TARGET("pclmul,avx,avx512f,avx512vl") +WOLFSSL_LOCAL void AES_GCM_encrypt_final_avx512(unsigned char* tag, + unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr) +{ + word64 rdi, rsi, rax, r10, r11, r9, r8, rsp, rdx, rcx, r13 = 0; + __m128i x0, x1, x4, x5, x6, x7, x8, x9, x10, x11; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rax = (word64)(word32)tbytes; + r10 = (word64)(word32)nbytes; + r11 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + r8 = (word64)(size_t)initCtr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_srli_epi64(x5, 63); + x7 = _mm_slli_epi64(x5, 1); + x8 = _mm_bslli_si128(x8, 8); + x7 = _mm_or_si128(x7, x8); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x7); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x1 = _mm_cvtsi64_si128((long long)rcx); + x0 = _mm_unpacklo_epi64(x0, x1); + x4 = _mm_xor_si128(x4, x0); + /* ghash_gfmul_red_avx */ + x8 = _mm_shuffle_epi32(x5, 0x4e); + x8 = _mm_xor_si128(x8, x5); + x9 = _mm_shuffle_epi32(x4, 0x4e); + x9 = _mm_xor_si128(x9, x4); + x7 = _mm_clmulepi64_si128(x4, x5, 0); + x10 = _mm_clmulepi64_si128(x4, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x9, 0); + x8 = _mm_ternarylogic_epi64(x8, x10, x7, 0x96); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x11 = _mm_clmulepi64_si128(x9, x7, 1); + x7 = _mm_shuffle_epi32(x7, 0x4e); + x8 = _mm_ternarylogic_epi64(x8, x7, x11, 0x96); + x11 = _mm_clmulepi64_si128(x9, x8, 1); + x8 = _mm_shuffle_epi32(x8, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x8, x11, 0x96); + x4 = x10; + x4 = _mm_shuffle_epi8(x4, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x4, x6); + if (((word32)rax) == (0x10)) { + goto L_AES_GCM_encrypt_final_avx512_store_tag_16; + } + rcx = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_encrypt_final_avx512_store_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + WC_S8(rsi, rcx) = (byte)((byte)r13); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)rax)) { + goto L_AES_GCM_encrypt_final_avx512_store_tag_loop; + } + goto L_AES_GCM_encrypt_final_avx512_store_tag_done; +L_AES_GCM_encrypt_final_avx512_store_tag_16: + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); +L_AES_GCM_encrypt_final_avx512_store_tag_done: + ; +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_GCM_decrypt_update_avx512(const unsigned char* key, + int nr, unsigned char* out, const unsigned char* in, unsigned int nbytes, + unsigned char* tag, unsigned char* h, unsigned char* counter) +{ + word64 rdi, rsi, r10, r11, r8, r9, rax, r12, rsp, r14, r13 = 0, r15 = 0, + rbx = 0, rcx = 0, rdx = 0; + __m128i x0 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), x5, x6, + x7 = _mm_setzero_si128(), x8; + __m512i z1 = _mm512_setzero_si512(), z2 = _mm512_setzero_si512(), + z3 = _mm512_setzero_si512(), z9, z10 = _mm512_setzero_si512(), + z11 = _mm512_setzero_si512(), z12 = _mm512_setzero_si512(), + z13 = _mm512_setzero_si512(), z14 = _mm512_setzero_si512(), + z15 = _mm512_setzero_si512(), z16 = _mm512_setzero_si512(), + z17 = _mm512_setzero_si512(), z18 = _mm512_setzero_si512(), + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(), + z23 = _mm512_setzero_si512(), z24 = _mm512_setzero_si512(), + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(), + z27 = _mm512_setzero_si512(), z28 = _mm512_setzero_si512(), + z29 = _mm512_setzero_si512(), z30 = _mm512_setzero_si512(), + z31 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[132]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + + rdi = (word64)(size_t)key; + rsi = (word64)(word32)nr; + r10 = (word64)(size_t)out; + r11 = (word64)(size_t)in; + r8 = (word64)(word64)nbytes; + r9 = (word64)(size_t)tag; + rax = (word64)(size_t)h; + r12 = (word64)(size_t)counter; + + rsp = (word64)(size_t)stk + 1056; + rsp = (word64)(rsp - 1040); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + z9 = _mm512_zextsi128_si512(_mm_srli_epi64(x5, 63)); + x8 = _mm_slli_epi64(x5, 1); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 8)); + x8 = _mm_or_si128(x8, _mm512_castsi512_si128(z9)); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x8); + r14 = (word32)(0); + if (((word32)r8) < (0x80)) { + goto L_AES_GCM_decrypt_update_avx512_done_128; + } + z2 = _mm512_zextsi128_si512(x6); + /* H ^ 1 */ + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x5); + /* H ^ 2 */ + x8 = _mm_clmulepi64_si128(x5, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x5, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x0 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x0); + /* H ^ 3 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x5, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x5)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + x0)); + x8 = _mm_clmulepi64_si128(x0, x5, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x5, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z1 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm512_castsi512_si128(z1)); + /* H ^ 4 */ + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + z3 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z11)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm512_castsi512_si128(z3)); + /* H ^ 5 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x7); + /* H ^ 6 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + /* H ^ 7 */ + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z1))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z3), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z3))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z1), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z1), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x7); + /* H ^ 8 */ + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z3), + _mm512_castsi512_si128(z3), 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z3), _mm512_castsi512_si128(z3), 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x7); + if (((word32)r8) < (0x100)) { + goto L_AES_GCM_decrypt_update_avx512_no_ext2; + } + /* H ^ 9 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 48)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 64))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 128), x7); + /* H ^ 10 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 144), x7); + /* H ^ 11 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 64)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 80))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 160), x7); + /* H ^ 12 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 176), x7); + /* H ^ 13 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 80)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 96))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 192), x7); + /* H ^ 14 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 208), x7); + /* H ^ 15 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 96)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 112))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 224), x7); + /* H ^ 16 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 240), x7); + if (((word32)r8) < (0x200)) { + goto L_AES_GCM_decrypt_update_avx512_no_ext; + } + /* H ^ 17 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 112)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 128))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 256), x7); + /* H ^ 18 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 272), x7); + /* H ^ 19 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 128)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 144))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 288), x7); + /* H ^ 20 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 304), x7); + /* H ^ 21 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 144)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 160))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 320), x7); + /* H ^ 22 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 160)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 336), x7); + /* H ^ 23 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 160)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 176))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 352), x7); + /* H ^ 24 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 176)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 368), x7); + /* H ^ 25 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 176)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 192))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 384), x7); + /* H ^ 26 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 192)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 400), x7); + /* H ^ 27 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 192)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 208))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 416), x7); + /* H ^ 28 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 208)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 432), x7); + /* H ^ 29 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 208)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 224))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 448), x7); + /* H ^ 30 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 224)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 464), x7); + /* H ^ 31 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 224)); + z1 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, + 240))); + /* ghash_gfmul_red_avx */ + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(x0, 0x4e)); + z9 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z9), x0)); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z1), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z1))); + x8 = _mm_clmulepi64_si128(_mm512_castsi512_si128(z1), x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z9), + _mm512_castsi512_si128(z10), 0)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), x8, 0x96)); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 480), x7); + /* H ^ 32 */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 240)); + x8 = _mm_clmulepi64_si128(x0, x0, 0); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x0, 0x11)); + z9 = _mm512_zextsi128_si512(_mm_setzero_si128()); + z10 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x8, 1)); + x8 = _mm_shuffle_epi32(x8, 0x4e); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), x8, _mm512_castsi512_si128(z12), 0x96)); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), _mm512_castsi512_si128(z9), 1)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z9), + 0x4e)); + z11 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z11), _mm512_castsi512_si128(z9), _mm512_castsi512_si128(z12), 0x96)); + x7 = _mm512_castsi512_si128(z11); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 496), x7); +L_AES_GCM_decrypt_update_avx512_no_ext: +L_AES_GCM_decrypt_update_avx512_no_ext2: + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_epi64, 0))); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + z1 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + z3 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + if (((word32)r8) < (0x200)) { + goto L_AES_GCM_decrypt_update_avx512_no_windows; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffe00); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 448)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 384)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 320)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 256)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + _mm512_storeu_si512((void*)WC_PW(rsp, 512), z23); + _mm512_storeu_si512((void*)WC_PW(rsp, 576), z24); + _mm512_storeu_si512((void*)WC_PW(rsp, 640), z25); + _mm512_storeu_si512((void*)WC_PW(rsp, 704), z26); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + _mm512_storeu_si512((void*)WC_PW(rsp, 768), z23); + _mm512_storeu_si512((void*)WC_PW(rsp, 832), z24); + _mm512_storeu_si512((void*)WC_PW(rsp, 896), z25); + _mm512_storeu_si512((void*)WC_PW(rsp, 960), z26); + /* 512 bytes of input */ + r15 = (word32)(0); + rbx = (word64)(r11 + r14); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 512)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 576)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 640)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 704)); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 768)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 832)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 896)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 960)); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 256)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 320)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 384)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 448)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + r14 = (word32)((word32)r14 + 0x200); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx512_last_aes; + } +L_AES_GCM_decrypt_update_avx512_win_loop: + rbx = (word64)(r11 + r14); + z21 = _mm512_setzero_si512(); + z21 = _mm512_inserti32x4(z21, x6, 0); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z31 = _mm512_loadu_si512((const void*)WC_PR(rbx, 0)); + z31 = _mm512_shuffle_epi8(z31, z30); + z31 = _mm512_xor_si512(z31, z21); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 512)), 0x11); + z27 = z23; + z28 = _mm512_xor_si512(z25, z24); + z29 = z26; + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z31 = _mm512_loadu_si512((const void*)WC_PR(rbx, 64)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 576)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z31 = _mm512_loadu_si512((const void*)WC_PR(rbx, 128)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 640)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z31 = _mm512_loadu_si512((const void*)WC_PR(rbx, 192)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 704)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf1 = (word32)rsi; + zf2 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_GCM_decrypt_update_avx512_a_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf3 = (word32)rsi; + zf4 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCM_decrypt_update_avx512_a_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_avx512_a_il_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r15); + rdx = (word64)(r10 + r15); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r15 = (word32)((word32)r15 + 0x100); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z31 = _mm512_loadu_si512((const void*)WC_PR(rbx, 256)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 768)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z31 = _mm512_loadu_si512((const void*)WC_PR(rbx, 320)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 832)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z31 = _mm512_loadu_si512((const void*)WC_PR(rbx, 384)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 896)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z31 = _mm512_loadu_si512((const void*)WC_PR(rbx, 448)); + z31 = _mm512_shuffle_epi8(z31, z30); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z23 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0); + z24 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 1); + z25 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0x10); + z26 = _mm512_clmulepi64_epi128(z31, _mm512_loadu_si512((const void*)WC_PR( + rsp, 960)), 0x11); + z27 = _mm512_xor_si512(z27, z23); + z28 = _mm512_ternarylogic_epi64(z28, z25, z24, 0x96); + z29 = _mm512_xor_si512(z29, z26); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf5 = (word32)rsi; + zf6 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCM_decrypt_update_avx512_b_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf7 = (word32)rsi; + zf8 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCM_decrypt_update_avx512_b_il_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_avx512_b_il_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r15); + rdx = (word64)(r10 + r15); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r15 = (word32)((word32)r15 + 0x100); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z23 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z23, 0x96); + z23 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z23, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + r14 = (word32)((word32)r14 + 0x200); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx512_win_loop; + } +L_AES_GCM_decrypt_update_avx512_last_aes: + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf9 = (word32)rsi; + zf10 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCM_decrypt_update_avx512_l1_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf11 = (word32)rsi; + zf12 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_GCM_decrypt_update_avx512_l1_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_avx512_l1_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r15); + rdx = (word64)(r10 + r15); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r15 = (word32)((word32)r15 + 0x100); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf13 = (word32)rsi; + zf14 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCM_decrypt_update_avx512_l2_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf15 = (word32)rsi; + zf16 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCM_decrypt_update_avx512_l2_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_avx512_l2_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r15); + rdx = (word64)(r10 + r15); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r15 = (word32)((word32)r15 + 0x100); +L_AES_GCM_decrypt_update_avx512_no_windows: + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 192)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 128)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z25 = _mm512_shuffle_i64x2(z25, z25, 0x1b); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z26 = _mm512_shuffle_i64x2(z26, z26, 0x1b); + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xffffff00); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx512_after_256; + } + /* 256 bytes of input */ + rbx = (word64)(r11 + r14); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 128)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z25, 0); + z17 = _mm512_clmulepi64_epi128(z21, z25, 1); + z18 = _mm512_clmulepi64_epi128(z21, z25, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z25, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 192)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z26, 0); + z17 = _mm512_clmulepi64_epi128(z21, z26, 1); + z18 = _mm512_clmulepi64_epi128(z21, z26, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z26, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + z18 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z2, 0))); + z18 = _mm512_shuffle_epi8(z18, z22); + z19 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z3, 0))); + z19 = _mm512_shuffle_epi8(z19, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_sixteen, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z18 = _mm512_xor_si512(z18, z9); + z19 = _mm512_xor_si512(z19, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z18 = _mm512_aesenc_epi128(z18, z10); + z19 = _mm512_aesenc_epi128(z19, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z18 = _mm512_aesenc_epi128(z18, z11); + z19 = _mm512_aesenc_epi128(z19, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z18 = _mm512_aesenc_epi128(z18, z12); + z19 = _mm512_aesenc_epi128(z19, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z18 = _mm512_aesenc_epi128(z18, z13); + z19 = _mm512_aesenc_epi128(z19, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z18 = _mm512_aesenc_epi128(z18, z14); + z19 = _mm512_aesenc_epi128(z19, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z18 = _mm512_aesenc_epi128(z18, z15); + z19 = _mm512_aesenc_epi128(z19, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z18 = _mm512_aesenc_epi128(z18, z1); + z19 = _mm512_aesenc_epi128(z19, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z18 = _mm512_aesenc_epi128(z18, z2); + z19 = _mm512_aesenc_epi128(z19, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + z18 = _mm512_aesenc_epi128(z18, z3); + z19 = _mm512_aesenc_epi128(z19, z3); + zf17 = (word32)rsi; + zf18 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_GCM_decrypt_update_avx512_t_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + zf19 = (word32)rsi; + zf20 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_GCM_decrypt_update_avx512_t_avx512_ctr16_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z18 = _mm512_aesenc_epi128(z18, z20); + z19 = _mm512_aesenc_epi128(z19, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_avx512_t_avx512_ctr16_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + z18 = _mm512_aesenclast_epi128(z18, z20); + z19 = _mm512_aesenclast_epi128(z19, z20); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z18 = _mm512_xor_si512(z18, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z19 = _mm512_xor_si512(z19, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z19); + r14 = (word32)((word32)r14 + 0x100); +L_AES_GCM_decrypt_update_avx512_after_256: + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xffffff80); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx512_after_128; + } + /* 128 bytes of input */ + rbx = (word64)(r11 + r14); + z23 = _mm512_loadu_si512((const void*)WC_PR(rsp, 64)); + z23 = _mm512_shuffle_i64x2(z23, z23, 0x1b); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsp, 0)); + z24 = _mm512_shuffle_i64x2(z24, z24, 0x1b); + z20 = _mm512_setzero_si512(); + z20 = _mm512_inserti32x4(z20, x6, 0); + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 0)); + z21 = _mm512_shuffle_epi8(z21, z30); + z21 = _mm512_xor_si512(z21, z20); + z16 = _mm512_clmulepi64_epi128(z21, z23, 0); + z17 = _mm512_clmulepi64_epi128(z21, z23, 1); + z18 = _mm512_clmulepi64_epi128(z21, z23, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z23, 0x11); + z27 = z16; + z28 = _mm512_xor_si512(z18, z17); + z29 = z19; + z21 = _mm512_loadu_si512((const void*)WC_PR(rbx, 64)); + z21 = _mm512_shuffle_epi8(z21, z30); + z16 = _mm512_clmulepi64_epi128(z21, z24, 0); + z17 = _mm512_clmulepi64_epi128(z21, z24, 1); + z18 = _mm512_clmulepi64_epi128(z21, z24, 0x10); + z19 = _mm512_clmulepi64_epi128(z21, z24, 0x11); + z27 = _mm512_xor_si512(z27, z16); + z28 = _mm512_ternarylogic_epi64(z28, z18, z17, 0x96); + z29 = _mm512_xor_si512(z29, z19); + z31 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + z21 = _mm512_clmulepi64_epi128(z31, z27, 1); + z27 = _mm512_shuffle_epi32(z27, 0x4e); + z28 = _mm512_ternarylogic_epi64(z28, z27, z21, 0x96); + z21 = _mm512_clmulepi64_epi128(z31, z28, 1); + z28 = _mm512_shuffle_epi32(z28, 0x4e); + z29 = _mm512_ternarylogic_epi64(z29, z28, z21, 0x96); + x0 = _mm512_extracti32x4_epi32(z29, 1); + x4 = _mm512_extracti32x4_epi32(z29, 2); + x5 = _mm512_extracti32x4_epi32(z29, 3); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z29), x0); + x6 = _mm_ternarylogic_epi64(x6, x5, x4, 0x96); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 64))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 80))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 96))); + z1 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 112))); + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 128))); + z3 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 144))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r12, + 0))); + z16 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z0, 0))); + z16 = _mm512_shuffle_epi8(z16, z22); + z17 = _mm512_add_epi32(z20, _mm512_loadu_si512((const void*)WC_PR( + L_avx512_aes_gcm_inc_z1, 0))); + z17 = _mm512_shuffle_epi8(z17, z22); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_eight, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x8); + z16 = _mm512_xor_si512(z16, z9); + z17 = _mm512_xor_si512(z17, z9); + z16 = _mm512_aesenc_epi128(z16, z10); + z17 = _mm512_aesenc_epi128(z17, z10); + z16 = _mm512_aesenc_epi128(z16, z11); + z17 = _mm512_aesenc_epi128(z17, z11); + z16 = _mm512_aesenc_epi128(z16, z12); + z17 = _mm512_aesenc_epi128(z17, z12); + z16 = _mm512_aesenc_epi128(z16, z13); + z17 = _mm512_aesenc_epi128(z17, z13); + z16 = _mm512_aesenc_epi128(z16, z14); + z17 = _mm512_aesenc_epi128(z17, z14); + z16 = _mm512_aesenc_epi128(z16, z15); + z17 = _mm512_aesenc_epi128(z17, z15); + z16 = _mm512_aesenc_epi128(z16, z1); + z17 = _mm512_aesenc_epi128(z17, z1); + z16 = _mm512_aesenc_epi128(z16, z2); + z17 = _mm512_aesenc_epi128(z17, z2); + z16 = _mm512_aesenc_epi128(z16, z3); + z17 = _mm512_aesenc_epi128(z17, z3); + zf21 = (word32)rsi; + zf22 = 0xb; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_GCM_decrypt_update_avx512_8_avx512_ctr8_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 176))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + zf23 = (word32)rsi; + zf24 = 0xd; + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_GCM_decrypt_update_avx512_8_avx512_ctr8_last; + } + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 208))); + z16 = _mm512_aesenc_epi128(z16, z20); + z17 = _mm512_aesenc_epi128(z17, z20); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_avx512_8_avx512_ctr8_last: + z16 = _mm512_aesenclast_epi128(z16, z20); + z17 = _mm512_aesenclast_epi128(z17, z20); + rcx = (word64)(r11 + r14); + rdx = (word64)(r10 + r14); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z16 = _mm512_xor_si512(z16, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z17 = _mm512_xor_si512(z17, z21); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z17); + r14 = (word32)((word32)r14 + 0x80); +L_AES_GCM_decrypt_update_avx512_after_128: + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); +L_AES_GCM_decrypt_update_avx512_done_128: + rdx = (word32)((word32)r8); + if (((word32)r14) >= ((word32)rdx)) { + goto L_AES_GCM_decrypt_update_avx512_done_dec; + } + r13 = (word32)((word32)r8); + r13 = (word32)((word32)r13 & 0xfffffff0); + if (((word32)r14) >= ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx512_last_block_done; + } +L_AES_GCM_decrypt_update_avx512_last_block_start: + z13 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r11, + r14))); + x0 = x5; + z1 = _mm512_zextsi128_si512(_mm_shuffle_epi8(_mm512_castsi512_si128(z13), + _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_bswap_mask, + 0)))); + z1 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), x6)); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x8 = _mm_shuffle_epi8(_mm512_castsi512_si128(z9), _mm_loadu_si128(( + const __m128i*)WC_PR(L_avx512_aes_gcm_bswap_epi64, 0))); + z9 = _mm512_zextsi128_si512(_mm_add_epi32(_mm512_castsi512_si128(z9), + _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_one, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), _mm512_castsi512_si128(z9)); + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + z10 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32))); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 1)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48))); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 64))); + z12 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z1), x0, 0)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 80))); + z1 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128(z1), + x0, 0x11)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 96))); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + z2 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 8)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z10), + 8)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 112))); + z2 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z2), + _mm512_castsi512_si128(z12))); + z3 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z1), + _mm512_castsi512_si128(z10))); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z2), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 128))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z2), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + z11 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(_mm512_castsi512_si128( + z10), x0, 0x10)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 144))); + z10 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z10), + 0x4e)); + z10 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z10), + _mm512_castsi512_si128(z11))); + x6 = _mm_xor_si128(_mm512_castsi512_si128(z10), _mm512_castsi512_si128(z3)); + zf25 = (word32)rsi; + zf26 = 0xb; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 160))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_GCM_decrypt_update_avx512_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 176))); + zf27 = (word32)rsi; + zf28 = 0xd; + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 192))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_GCM_decrypt_update_avx512_aesenc_gfmul_last; + } + x8 = _mm_aesenc_si128(x8, _mm512_castsi512_si128(z9)); + x8 = _mm_aesenc_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 208))); + z9 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 224))); +L_AES_GCM_decrypt_update_avx512_aesenc_gfmul_last: + x8 = _mm_aesenclast_si128(x8, _mm512_castsi512_si128(z9)); + x0 = _mm512_castsi512_si128(z13); + x8 = _mm_xor_si128(x8, x0); + _mm_storeu_si128((__m128i*)WC_PW(r10, r14), x8); + r14 = (word32)((word32)r14 + 0x10); + if (((word32)r14) < ((word32)r13)) { + goto L_AES_GCM_decrypt_update_avx512_last_block_start; + } +L_AES_GCM_decrypt_update_avx512_last_block_done: +L_AES_GCM_decrypt_update_avx512_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x6); +} + +WC_X64I_TARGET("pclmul,avx,avx512f,avx512vl") +WOLFSSL_LOCAL void AES_GCM_decrypt_final_avx512(unsigned char* tag, + const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes, + unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res) +{ + word64 rdi, rsi, rax, r10, r11, r9, r8, rbp, rsp, rdx, rcx, r12 = 0, + r13 = 0; + __m128i x0, x1, x5, x6, x7, x8, x9, x10, x11, x15; + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)tag; + rsi = (word64)(size_t)authTag; + rax = (word64)(word32)tbytes; + r10 = (word64)(word32)nbytes; + r11 = (word64)(word32)abytes; + r9 = (word64)(size_t)h; + r8 = (word64)(size_t)initCtr; + rbp = (word64)(size_t)res; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_srli_epi64(x5, 63); + x7 = _mm_slli_epi64(x5, 1); + x8 = _mm_bslli_si128(x8, 8); + x7 = _mm_or_si128(x7, x8); + x5 = _mm_shuffle_epi32(x5, 0xff); + x5 = _mm_srai_epi32(x5, 31); + x5 = _mm_and_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_mod2_128, 0))); + x5 = _mm_xor_si128(x5, x7); + rdx = (word32)((word32)r10); + rcx = (word32)((word32)r11); + rdx = (word64)(rdx << 3); + rcx = (word64)(rcx << 3); + x0 = _mm_cvtsi64_si128((long long)rdx); + x1 = _mm_cvtsi64_si128((long long)rcx); + x0 = _mm_unpacklo_epi64(x0, x1); + x6 = _mm_xor_si128(x6, x0); + /* ghash_gfmul_red_avx */ + x8 = _mm_shuffle_epi32(x5, 0x4e); + x8 = _mm_xor_si128(x8, x5); + x9 = _mm_shuffle_epi32(x6, 0x4e); + x9 = _mm_xor_si128(x9, x6); + x7 = _mm_clmulepi64_si128(x6, x5, 0); + x10 = _mm_clmulepi64_si128(x6, x5, 0x11); + x8 = _mm_clmulepi64_si128(x8, x9, 0); + x8 = _mm_ternarylogic_epi64(x8, x10, x7, 0x96); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_gcm_mod2_128, 0)); + x11 = _mm_clmulepi64_si128(x9, x7, 1); + x7 = _mm_shuffle_epi32(x7, 0x4e); + x8 = _mm_ternarylogic_epi64(x8, x7, x11, 0x96); + x11 = _mm_clmulepi64_si128(x9, x8, 1); + x8 = _mm_shuffle_epi32(x8, 0x4e); + x10 = _mm_ternarylogic_epi64(x10, x8, x11, 0x96); + x6 = x10; + x6 = _mm_shuffle_epi8(x6, _mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_gcm_bswap_mask, 0))); + x0 = _mm_xor_si128(x6, x15); + if (((word32)rax) == (0x10)) { + goto L_AES_GCM_decrypt_final_avx512_cmp_tag_16; + } + rsp = (word64)(rsp - 16); + rcx = (word64)(0); + r12 = (word64)(0); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x0); +L_AES_GCM_decrypt_final_avx512_cmp_tag_loop: + r13 = (word32)((word32)WC_L8(rsp, rcx)); + r13 = (r13 & ~(word64)0xff) | ((word64)(byte)((byte)r13 ^ WC_L8(rsi, + rcx)) & 0xff); + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)((byte)r12 | ( + byte)r13) & 0xff); + rcx = (word32)((word32)rcx + 1); + if (((word32)rcx) != ((word32)rax)) { + goto L_AES_GCM_decrypt_final_avx512_cmp_tag_loop; + } + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)(((byte)r12) == (0)) & 0xff); + rsp = (word64)(rsp + 16); + rcx = (word64)(0); + goto L_AES_GCM_decrypt_final_avx512_cmp_tag_done; +L_AES_GCM_decrypt_final_avx512_cmp_tag_16: + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_cmpeq_epi8(x0, x1); + rdx = (word32)((word32)_mm_movemask_epi8(x0)); + /* %%edx == 0xFFFF then return 1 else => return 0 */ + r12 = (word32)(0); + r12 = (r12 & ~(word64)0xff) | ((word64)(byte)(((word32)rdx) == ( + 0xffff)) & 0xff); +L_AES_GCM_decrypt_final_avx512_cmp_tag_done: + WC_S32(rbp, 0) = (word32)((word32)r12); +} + +#endif /* WOLFSSL_AESGCM_STREAM */ +#ifdef WOLFSSL_AESGCM_SIV +XALIGNED(16) static const word64 L_aes_gcm_siv_bswap_mask_avx512[] + WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +WC_X64I_TARGET("pclmul,vpclmulqdq,avx,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_GCMSIV_polyval_avx512(unsigned char* s, + const unsigned char* h, const unsigned char* data, unsigned int blocks) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10 = 0; + __m128i x0, x1, x17, x18, x19, x20, x21 = _mm_setzero_si128(), + x22 = _mm_setzero_si128(); + __m512i z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), z6, z7, + z8, z9, z10, z11, z12 = _mm512_setzero_si512(), + z13 = _mm512_setzero_si512(), z14 = _mm512_setzero_si512(), + z15 = _mm512_setzero_si512(), z16 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)h; + rdx = (word64)(size_t)data; + rcx = (word64)(word32)blocks; + + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_siv_bswap_mask_avx512, 0))); + x20 = x1; + z2 = _mm512_inserti32x4(z2, x20, 3); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z2 = _mm512_inserti32x4(z2, x20, 2); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z2 = _mm512_inserti32x4(z2, x20, 1); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z2 = _mm512_inserti32x4(z2, x20, 0); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z3 = _mm512_inserti32x4(z3, x20, 3); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z3 = _mm512_inserti32x4(z3, x20, 2); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z3 = _mm512_inserti32x4(z3, x20, 1); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z3 = _mm512_inserti32x4(z3, x20, 0); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z4 = _mm512_inserti32x4(z4, x20, 3); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z4 = _mm512_inserti32x4(z4, x20, 2); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z4 = _mm512_inserti32x4(z4, x20, 1); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z4 = _mm512_inserti32x4(z4, x20, 0); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z5 = _mm512_inserti32x4(z5, x20, 3); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z5 = _mm512_inserti32x4(z5, x20, 2); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z5 = _mm512_inserti32x4(z5, x20, 1); + x17 = _mm_clmulepi64_si128(x20, x1, 0); + x18 = _mm_clmulepi64_si128(x20, x1, 0x11); + x19 = _mm_clmulepi64_si128(x20, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x20, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x20 = _mm_xor_si128(x18, x17); + z5 = _mm512_inserti32x4(z5, x20, 0); + r8 = (word32)((word32)rcx); + r8 = (word32)((word32)r8 << 4); + rax = (word32)(0); + r9 = (word32)((word32)r8); + r9 = (word32)((word32)r9 & 0xffffff00); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_GCMSIV_polyval_avx512_sixteen_done; + } +L_AES_GCMSIV_polyval_avx512_sixteen: + r10 = (word64)(rdx + rax); + z6 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z8 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z9 = _mm512_loadu_si512((const void*)WC_PR(r10, 192)); + z16 = _mm512_setzero_si512(); + z16 = _mm512_inserti32x4(z16, x0, 0); + z6 = _mm512_xor_si512(z6, z16); + z10 = _mm512_clmulepi64_epi128(z6, z5, 0); + z11 = _mm512_clmulepi64_epi128(z6, z5, 0x11); + z14 = _mm512_clmulepi64_epi128(z6, z5, 0x10); + z15 = _mm512_clmulepi64_epi128(z6, z5, 1); + z14 = _mm512_xor_si512(z14, z15); + z15 = _mm512_bslli_epi128(z14, 8); + z14 = _mm512_bsrli_epi128(z14, 8); + z10 = _mm512_xor_si512(z10, z15); + z11 = _mm512_xor_si512(z11, z14); + z12 = _mm512_clmulepi64_epi128(z7, z4, 0); + z13 = _mm512_clmulepi64_epi128(z7, z4, 0x11); + z14 = _mm512_clmulepi64_epi128(z7, z4, 0x10); + z15 = _mm512_clmulepi64_epi128(z7, z4, 1); + z14 = _mm512_xor_si512(z14, z15); + z15 = _mm512_bslli_epi128(z14, 8); + z14 = _mm512_bsrli_epi128(z14, 8); + z12 = _mm512_xor_si512(z12, z15); + z13 = _mm512_xor_si512(z13, z14); + z10 = _mm512_xor_si512(z10, z12); + z11 = _mm512_xor_si512(z11, z13); + z12 = _mm512_clmulepi64_epi128(z8, z3, 0); + z13 = _mm512_clmulepi64_epi128(z8, z3, 0x11); + z14 = _mm512_clmulepi64_epi128(z8, z3, 0x10); + z15 = _mm512_clmulepi64_epi128(z8, z3, 1); + z14 = _mm512_xor_si512(z14, z15); + z15 = _mm512_bslli_epi128(z14, 8); + z14 = _mm512_bsrli_epi128(z14, 8); + z12 = _mm512_xor_si512(z12, z15); + z13 = _mm512_xor_si512(z13, z14); + z10 = _mm512_xor_si512(z10, z12); + z11 = _mm512_xor_si512(z11, z13); + z12 = _mm512_clmulepi64_epi128(z9, z2, 0); + z13 = _mm512_clmulepi64_epi128(z9, z2, 0x11); + z14 = _mm512_clmulepi64_epi128(z9, z2, 0x10); + z15 = _mm512_clmulepi64_epi128(z9, z2, 1); + z14 = _mm512_xor_si512(z14, z15); + z15 = _mm512_bslli_epi128(z14, 8); + z14 = _mm512_bsrli_epi128(z14, 8); + z12 = _mm512_xor_si512(z12, z15); + z13 = _mm512_xor_si512(z13, z14); + z10 = _mm512_xor_si512(z10, z12); + z11 = _mm512_xor_si512(z11, z13); + x19 = _mm512_extracti32x4_epi32(z10, 1); + x21 = _mm512_extracti32x4_epi32(z10, 2); + x22 = _mm512_extracti32x4_epi32(z10, 3); + x17 = _mm_xor_si128(_mm512_castsi512_si128(z10), x19); + x17 = _mm_ternarylogic_epi64(x17, x22, x21, 0x96); + x19 = _mm512_extracti32x4_epi32(z11, 1); + x21 = _mm512_extracti32x4_epi32(z11, 2); + x22 = _mm512_extracti32x4_epi32(z11, 3); + x18 = _mm_xor_si128(_mm512_castsi512_si128(z11), x19); + x18 = _mm_ternarylogic_epi64(x18, x22, x21, 0x96); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x0 = _mm_xor_si128(x18, x17); + rax = (word32)((word32)rax + 0x100); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_GCMSIV_polyval_avx512_sixteen; + } +L_AES_GCMSIV_polyval_avx512_sixteen_done: + r9 = (word32)((word32)r8); + r9 = (word32)((word32)r9 & 0xffffff80); + if (((word32)rax) >= ((word32)r9)) { + goto L_AES_GCMSIV_polyval_avx512_eight_done; + } + r10 = (word64)(rdx + rax); + z6 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z16 = _mm512_setzero_si512(); + z16 = _mm512_inserti32x4(z16, x0, 0); + z6 = _mm512_xor_si512(z6, z16); + z10 = _mm512_clmulepi64_epi128(z6, z3, 0); + z11 = _mm512_clmulepi64_epi128(z6, z3, 0x11); + z14 = _mm512_clmulepi64_epi128(z6, z3, 0x10); + z15 = _mm512_clmulepi64_epi128(z6, z3, 1); + z14 = _mm512_xor_si512(z14, z15); + z15 = _mm512_bslli_epi128(z14, 8); + z14 = _mm512_bsrli_epi128(z14, 8); + z10 = _mm512_xor_si512(z10, z15); + z11 = _mm512_xor_si512(z11, z14); + z12 = _mm512_clmulepi64_epi128(z7, z2, 0); + z13 = _mm512_clmulepi64_epi128(z7, z2, 0x11); + z14 = _mm512_clmulepi64_epi128(z7, z2, 0x10); + z15 = _mm512_clmulepi64_epi128(z7, z2, 1); + z14 = _mm512_xor_si512(z14, z15); + z15 = _mm512_bslli_epi128(z14, 8); + z14 = _mm512_bsrli_epi128(z14, 8); + z12 = _mm512_xor_si512(z12, z15); + z13 = _mm512_xor_si512(z13, z14); + z10 = _mm512_xor_si512(z10, z12); + z11 = _mm512_xor_si512(z11, z13); + x19 = _mm512_extracti32x4_epi32(z10, 1); + x21 = _mm512_extracti32x4_epi32(z10, 2); + x22 = _mm512_extracti32x4_epi32(z10, 3); + x17 = _mm_xor_si128(_mm512_castsi512_si128(z10), x19); + x17 = _mm_ternarylogic_epi64(x17, x22, x21, 0x96); + x19 = _mm512_extracti32x4_epi32(z11, 1); + x21 = _mm512_extracti32x4_epi32(z11, 2); + x22 = _mm512_extracti32x4_epi32(z11, 3); + x18 = _mm_xor_si128(_mm512_castsi512_si128(z11), x19); + x18 = _mm_ternarylogic_epi64(x18, x22, x21, 0x96); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x0 = _mm_xor_si128(x18, x17); + rax = (word32)((word32)rax + 0x80); +L_AES_GCMSIV_polyval_avx512_eight_done: +L_AES_GCMSIV_polyval_avx512_rem: + if (((word32)rax) >= ((word32)r8)) { + goto L_AES_GCMSIV_polyval_avx512_done; + } + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, rax))); + x17 = _mm_clmulepi64_si128(x0, x1, 0); + x18 = _mm_clmulepi64_si128(x0, x1, 0x11); + x19 = _mm_clmulepi64_si128(x0, x1, 0x10); + z6 = _mm512_zextsi128_si512(_mm_clmulepi64_si128(x0, x1, 1)); + x19 = _mm_xor_si128(x19, _mm512_castsi512_si128(z6)); + z6 = _mm512_zextsi128_si512(_mm_bslli_si128(x19, 8)); + x19 = _mm_bsrli_si128(x19, 8); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x18 = _mm_xor_si128(x18, x19); + z9 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_srli_epi32(x18, 31)); + x17 = _mm_slli_epi32(x17, 1); + x18 = _mm_slli_epi32(x18, 1); + z11 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 12)); + z10 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z10), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), 4)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + x18 = _mm_ternarylogic_epi64(x18, _mm512_castsi512_si128(z11), + _mm512_castsi512_si128(z10), 0x96); + z9 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 31)); + z10 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 30)); + z11 = _mm512_zextsi128_si512(_mm_slli_epi32(x17, 25)); + z9 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z9), _mm512_castsi512_si128(z11), _mm512_castsi512_si128(z10), 0x96)); + z10 = _mm512_zextsi128_si512(_mm_bsrli_si128(_mm512_castsi512_si128(z9), + 4)); + z9 = _mm512_zextsi128_si512(_mm_bslli_si128(_mm512_castsi512_si128(z9), + 12)); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z9)); + z6 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 1)); + z7 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 2)); + z8 = _mm512_zextsi128_si512(_mm_srli_epi32(x17, 7)); + z6 = _mm512_zextsi128_si512(_mm_ternarylogic_epi64(_mm512_castsi512_si128( + z6), _mm512_castsi512_si128(z8), _mm512_castsi512_si128(z7), 0x96)); + z6 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z6), + _mm512_castsi512_si128(z10))); + x17 = _mm_xor_si128(x17, _mm512_castsi512_si128(z6)); + x0 = _mm_xor_si128(x18, x17); + rax = (word32)((word32)rax + 0x10); + goto L_AES_GCMSIV_polyval_avx512_rem; +L_AES_GCMSIV_polyval_avx512_done: + x0 = _mm_shuffle_epi8(x0, _mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcm_siv_bswap_mask_avx512, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); +} + +XALIGNED(32) static const word64 L_aes_gcmsiv_ctr_inc_avx512[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0x0000000000000002ULL, 0x0000000000000000ULL, + 0x0000000000000003ULL, 0x0000000000000000ULL, + 0x0000000000000004ULL, 0x0000000000000000ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000000000008ULL, 0x0000000000000000ULL, + 0x0000000000000009ULL, 0x0000000000000000ULL, + 0x000000000000000aULL, 0x0000000000000000ULL, + 0x000000000000000bULL, 0x0000000000000000ULL, + 0x000000000000000cULL, 0x0000000000000000ULL, + 0x000000000000000dULL, 0x0000000000000000ULL, + 0x000000000000000eULL, 0x0000000000000000ULL, + 0x000000000000000fULL, 0x0000000000000000ULL, + 0x0000000000000010ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("aes,vaes,avx,avx512f") +WOLFSSL_LOCAL void AES_GCMSIV_ctr_avx512(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10 = 0, r11 = 0, r12 = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7, z8, z9, z10, + z11 = _mm512_setzero_si512(), z12 = _mm512_setzero_si512(), + z13 = _mm512_setzero_si512(), z14 = _mm512_setzero_si512(), + z15 = _mm512_setzero_si512(), z16 = _mm512_setzero_si512(), + z17 = _mm512_setzero_si512(), z18 = _mm512_setzero_si512(), + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(), + z23 = _mm512_setzero_si512(), z24 = _mm512_setzero_si512(), + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + r9 = (word64)(size_t)ctr; + + z7 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + z8 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcmsiv_ctr_inc_avx512, 256))); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcmsiv_ctr_inc_avx512, 64))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_gcmsiv_ctr_inc_avx512, 16))); + rax = (word32)(0); + if (((word32)rdx) < (0x40)) { + goto L_AES_GCMSIV_ctr_avx512_done_64; + } + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + z16 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + z17 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + z18 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + z19 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + z21 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)((word32)r8) < (sword32)(0xb)) { + goto L_AES_GCMSIV_ctr_avx512_key_cached; + } + z23 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + z24 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)((word32)r8) < (sword32)(0xd)) { + goto L_AES_GCMSIV_ctr_avx512_key_cached; + } + z25 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + z26 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_GCMSIV_ctr_avx512_key_cached: + zf1 = (word32)rdx; + zf2 = 0x100; + r10 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_GCMSIV_ctr_avx512_done_256; + } + r10 = (word32)((word32)r10 & 0xffffff00); + z4 = _mm512_add_epi32(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_gcmsiv_ctr_inc_avx512, 0))); + z5 = _mm512_add_epi32(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_gcmsiv_ctr_inc_avx512, 64))); + z6 = _mm512_add_epi32(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_gcmsiv_ctr_inc_avx512, 128))); + z7 = _mm512_add_epi32(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_gcmsiv_ctr_inc_avx512, 192))); +L_AES_GCMSIV_ctr_avx512_enc_256: + /* 256 bytes of input */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + z0 = z4; + z1 = z5; + z2 = z6; + z3 = z7; + z4 = _mm512_add_epi32(z4, z8); + z5 = _mm512_add_epi32(z5, z8); + z6 = _mm512_add_epi32(z6, z8); + z7 = _mm512_add_epi32(z7, z8); + /* aes_enc_block */ + z0 = _mm512_xor_si512(z0, z12); + z1 = _mm512_xor_si512(z1, z12); + z2 = _mm512_xor_si512(z2, z12); + z3 = _mm512_xor_si512(z3, z12); + z0 = _mm512_aesenc_epi128(z0, z13); + z1 = _mm512_aesenc_epi128(z1, z13); + z2 = _mm512_aesenc_epi128(z2, z13); + z3 = _mm512_aesenc_epi128(z3, z13); + z0 = _mm512_aesenc_epi128(z0, z14); + z1 = _mm512_aesenc_epi128(z1, z14); + z2 = _mm512_aesenc_epi128(z2, z14); + z3 = _mm512_aesenc_epi128(z3, z14); + z0 = _mm512_aesenc_epi128(z0, z15); + z1 = _mm512_aesenc_epi128(z1, z15); + z2 = _mm512_aesenc_epi128(z2, z15); + z3 = _mm512_aesenc_epi128(z3, z15); + z0 = _mm512_aesenc_epi128(z0, z16); + z1 = _mm512_aesenc_epi128(z1, z16); + z2 = _mm512_aesenc_epi128(z2, z16); + z3 = _mm512_aesenc_epi128(z3, z16); + z0 = _mm512_aesenc_epi128(z0, z17); + z1 = _mm512_aesenc_epi128(z1, z17); + z2 = _mm512_aesenc_epi128(z2, z17); + z3 = _mm512_aesenc_epi128(z3, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z1 = _mm512_aesenc_epi128(z1, z18); + z2 = _mm512_aesenc_epi128(z2, z18); + z3 = _mm512_aesenc_epi128(z3, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z1 = _mm512_aesenc_epi128(z1, z19); + z2 = _mm512_aesenc_epi128(z2, z19); + z3 = _mm512_aesenc_epi128(z3, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z1 = _mm512_aesenc_epi128(z1, z20); + z2 = _mm512_aesenc_epi128(z2, z20); + z3 = _mm512_aesenc_epi128(z3, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z1 = _mm512_aesenc_epi128(z1, z21); + z2 = _mm512_aesenc_epi128(z2, z21); + z3 = _mm512_aesenc_epi128(z3, z21); + zf3 = (word32)r8; + zf4 = 0xb; + z11 = z22; + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_GCMSIV_ctr_avx512_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z22); + z1 = _mm512_aesenc_epi128(z1, z22); + z2 = _mm512_aesenc_epi128(z2, z22); + z3 = _mm512_aesenc_epi128(z3, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z1 = _mm512_aesenc_epi128(z1, z23); + z2 = _mm512_aesenc_epi128(z2, z23); + z3 = _mm512_aesenc_epi128(z3, z23); + zf5 = (word32)r8; + zf6 = 0xd; + z11 = z24; + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_GCMSIV_ctr_avx512_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z24); + z1 = _mm512_aesenc_epi128(z1, z24); + z2 = _mm512_aesenc_epi128(z2, z24); + z3 = _mm512_aesenc_epi128(z3, z24); + z0 = _mm512_aesenc_epi128(z0, z25); + z1 = _mm512_aesenc_epi128(z1, z25); + z2 = _mm512_aesenc_epi128(z2, z25); + z3 = _mm512_aesenc_epi128(z3, z25); + z11 = z26; +L_AES_GCMSIV_ctr_avx512_256_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z11); + z1 = _mm512_aesenclast_epi128(z1, z11); + z2 = _mm512_aesenclast_epi128(z2, z11); + z3 = _mm512_aesenclast_epi128(z3, z11); + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(r11, 0))); + z1 = _mm512_xor_si512(z1, _mm512_loadu_si512((const void*)WC_PR(r11, 64))); + z2 = _mm512_xor_si512(z2, _mm512_loadu_si512((const void*)WC_PR(r11, 128))); + z3 = _mm512_xor_si512(z3, _mm512_loadu_si512((const void*)WC_PR(r11, 192))); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z1); + _mm512_storeu_si512((void*)WC_PW(r12, 128), z2); + _mm512_storeu_si512((void*)WC_PW(r12, 192), z3); + rax = (word32)((word32)rax + 0x100); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_avx512_enc_256; + } + z7 = _mm512_shuffle_i64x2(z4, z4, 0); +L_AES_GCMSIV_ctr_avx512_done_256: + r10 = (word32)((word32)rdx); + r10 = (word32)((word32)r10 & 0xffffffc0); + if (((word32)rax) == ((word32)r10)) { + goto L_AES_GCMSIV_ctr_avx512_done_64; + } +L_AES_GCMSIV_ctr_avx512_enc_64: + /* 64 bytes of input */ + /* siv_ctr_enc_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + z0 = _mm512_add_epi32(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_gcmsiv_ctr_inc_avx512, 0))); + z7 = _mm512_add_epi32(z7, z9); + /* aes_enc_block */ + z0 = _mm512_xor_si512(z0, z12); + z0 = _mm512_aesenc_epi128(z0, z13); + z0 = _mm512_aesenc_epi128(z0, z14); + z0 = _mm512_aesenc_epi128(z0, z15); + z0 = _mm512_aesenc_epi128(z0, z16); + z0 = _mm512_aesenc_epi128(z0, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + zf7 = (word32)r8; + zf8 = 0xb; + z11 = z22; + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_GCMSIV_ctr_avx512_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + zf9 = (word32)r8; + zf10 = 0xd; + z11 = z24; + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_GCMSIV_ctr_avx512_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z24); + z0 = _mm512_aesenc_epi128(z0, z25); + z11 = z26; +L_AES_GCMSIV_ctr_avx512_64_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z11); + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(r11, 0))); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z0); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_avx512_enc_64; + } +L_AES_GCMSIV_ctr_avx512_done_64: + zf11 = (word32)rax; + zf12 = (word32)rdx; + r10 = (word32)((word32)rdx); + if ((zf11) == (zf12)) { + goto L_AES_GCMSIV_ctr_avx512_done_enc; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_GCMSIV_ctr_avx512_enc_16: + /* 16 bytes of input */ + z0 = _mm512_zextsi128_si512(_mm512_castsi512_si128(z7)); + z7 = _mm512_add_epi32(z7, z10); + /* aes_enc_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf13 = (word32)r8; + zf14 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_GCMSIV_ctr_avx512_16_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf15 = (word32)r8; + zf16 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_GCMSIV_ctr_avx512_16_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_GCMSIV_ctr_avx512_16_aes_enc_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + r11 = (word64)(rdi + rax); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)))); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), _mm512_castsi512_si128(z0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_GCMSIV_ctr_avx512_enc_16; + } +L_AES_GCMSIV_ctr_avx512_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z7)); +} + +#endif /* WOLFSSL_AESGCM_SIV */ +#endif /* HAVE_INTEL_AVX512 */ +#endif /* WOLFSSL_X86_64_BUILD */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/aes_x86_64_intrin.c b/wolfcrypt/src/aes_x86_64_intrin.c new file mode 100644 index 00000000000..4a64a228341 --- /dev/null +++ b/wolfcrypt/src/aes_x86_64_intrin.c @@ -0,0 +1,6652 @@ +/* aes_x86_64_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_AES_X86_64_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_X86_64_BUILD +extern WOLFSSL_LOCAL void AES_128_Key_Expansion_AESNI( + const unsigned char* userkey, unsigned char* key_schedule); +extern WOLFSSL_LOCAL void AES_192_Key_Expansion_AESNI( + const unsigned char* userkey, unsigned char* key_schedule); +extern WOLFSSL_LOCAL void AES_256_Key_Expansion_AESNI( + const unsigned char* userkey, unsigned char* key_schedule); +extern WOLFSSL_LOCAL void AES_ECB_encrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_ECB_decrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CBC_encrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CBC_decrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CTR_encrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr); +#ifdef HAVE_INTEL_AVX1 +extern WOLFSSL_LOCAL void AES_ECB_encrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_ECB_decrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CBC_encrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CBC_decrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CTR_encrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr); +#endif +#ifdef HAVE_INTEL_VAES +extern WOLFSSL_LOCAL void AES_ECB_encrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_ECB_decrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CBC_encrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CBC_decrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CTR_encrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr); +#endif +#ifdef HAVE_INTEL_AVX512 +extern WOLFSSL_LOCAL void AES_ECB_encrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_ECB_decrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CBC_encrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CBC_decrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr); +extern WOLFSSL_LOCAL void AES_CTR_encrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr); + +#endif +#endif +#ifdef WOLFSSL_X86_64_BUILD +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_128_Key_Expansion_AESNI(const unsigned char* userkey, + unsigned char* key_schedule) +{ + word64 rdi, rsi; + __m128i x0, x1, x2; + + rdi = (word64)(size_t)userkey; + rsi = (word64)(size_t)key_schedule; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); + x1 = _mm_aeskeygenassist_si128(x0, 1); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x0); + x1 = _mm_aeskeygenassist_si128(x0, 2); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x0); + x1 = _mm_aeskeygenassist_si128(x0, 4); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x0); + x1 = _mm_aeskeygenassist_si128(x0, 8); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x0); + x1 = _mm_aeskeygenassist_si128(x0, 0x10); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x0); + x1 = _mm_aeskeygenassist_si128(x0, 0x20); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x0); + x1 = _mm_aeskeygenassist_si128(x0, 0x40); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x0); + x1 = _mm_aeskeygenassist_si128(x0, 0x80); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 128), x0); + x1 = _mm_aeskeygenassist_si128(x0, 0x1b); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 144), x0); + x1 = _mm_aeskeygenassist_si128(x0, 0x36); + x1 = _mm_shuffle_epi32(x1, 0xff); + x2 = x0; + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_bslli_si128(x2, 4); + x0 = _mm_xor_si128(x0, x2); + x0 = _mm_xor_si128(x0, x1); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 160), x0); +} + +WC_X64I_TARGET("sse2,sse4.1,aes") +WOLFSSL_LOCAL void AES_192_Key_Expansion_AESNI(const unsigned char* userkey, + unsigned char* key_schedule) +{ + word64 rdi, rsi; + __m128i x0, x1, x2, x3, x4, x5; + + rdi = (word64)(size_t)userkey; + rsi = (word64)(size_t)key_schedule; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_setzero_si128(); + x1 = _mm_insert_epi64(x1, (long long)WC_L64(rdi, 16), 0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); + x4 = x1; + x2 = _mm_aeskeygenassist_si128(x1, 1); + x2 = _mm_shuffle_epi32(x2, 0x55); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_shuffle_epi32(x0, 0xff); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(x4), _mm_castsi128_pd( + x0), 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x4); + x5 = x0; + x5 = _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(x5), _mm_castsi128_pd( + x1), 1)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x5); + x2 = _mm_aeskeygenassist_si128(x1, 2); + x2 = _mm_shuffle_epi32(x2, 0x55); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_shuffle_epi32(x0, 0xff); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x0); + x4 = x1; + x2 = _mm_aeskeygenassist_si128(x1, 4); + x2 = _mm_shuffle_epi32(x2, 0x55); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_shuffle_epi32(x0, 0xff); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(x4), _mm_castsi128_pd( + x0), 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x4); + x5 = x0; + x5 = _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(x5), _mm_castsi128_pd( + x1), 1)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x5); + x2 = _mm_aeskeygenassist_si128(x1, 8); + x2 = _mm_shuffle_epi32(x2, 0x55); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_shuffle_epi32(x0, 0xff); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x0); + x4 = x1; + x2 = _mm_aeskeygenassist_si128(x1, 0x10); + x2 = _mm_shuffle_epi32(x2, 0x55); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_shuffle_epi32(x0, 0xff); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(x4), _mm_castsi128_pd( + x0), 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x4); + x5 = x0; + x5 = _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(x5), _mm_castsi128_pd( + x1), 1)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 128), x5); + x2 = _mm_aeskeygenassist_si128(x1, 0x20); + x2 = _mm_shuffle_epi32(x2, 0x55); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_shuffle_epi32(x0, 0xff); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 144), x0); + x4 = x1; + x2 = _mm_aeskeygenassist_si128(x1, 0x40); + x2 = _mm_shuffle_epi32(x2, 0x55); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_shuffle_epi32(x0, 0xff); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(x4), _mm_castsi128_pd( + x0), 0)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 160), x4); + x5 = x0; + x5 = _mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(x5), _mm_castsi128_pd( + x1), 1)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 176), x5); + x2 = _mm_aeskeygenassist_si128(x1, 0x80); + x2 = _mm_shuffle_epi32(x2, 0x55); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + x2 = _mm_shuffle_epi32(x0, 0xff); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 192), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 208), x1); +} + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_256_Key_Expansion_AESNI(const unsigned char* userkey, + unsigned char* key_schedule) +{ + word64 rdi, rsi; + __m128i x0, x1, x2, x3; + + rdi = (word64)(size_t)userkey; + rsi = (word64)(size_t)key_schedule; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x1); + x2 = _mm_aeskeygenassist_si128(x1, 1); + x2 = _mm_shuffle_epi32(x2, 0xff); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x0); + x2 = _mm_aeskeygenassist_si128(x0, 0); + x2 = _mm_shuffle_epi32(x2, 0xaa); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x1); + x2 = _mm_aeskeygenassist_si128(x1, 2); + x2 = _mm_shuffle_epi32(x2, 0xff); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x0); + x2 = _mm_aeskeygenassist_si128(x0, 0); + x2 = _mm_shuffle_epi32(x2, 0xaa); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x1); + x2 = _mm_aeskeygenassist_si128(x1, 4); + x2 = _mm_shuffle_epi32(x2, 0xff); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x0); + x2 = _mm_aeskeygenassist_si128(x0, 0); + x2 = _mm_shuffle_epi32(x2, 0xaa); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x1); + x2 = _mm_aeskeygenassist_si128(x1, 8); + x2 = _mm_shuffle_epi32(x2, 0xff); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 128), x0); + x2 = _mm_aeskeygenassist_si128(x0, 0); + x2 = _mm_shuffle_epi32(x2, 0xaa); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 144), x1); + x2 = _mm_aeskeygenassist_si128(x1, 0x10); + x2 = _mm_shuffle_epi32(x2, 0xff); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 160), x0); + x2 = _mm_aeskeygenassist_si128(x0, 0); + x2 = _mm_shuffle_epi32(x2, 0xaa); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 176), x1); + x2 = _mm_aeskeygenassist_si128(x1, 0x20); + x2 = _mm_shuffle_epi32(x2, 0xff); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 192), x0); + x2 = _mm_aeskeygenassist_si128(x0, 0); + x2 = _mm_shuffle_epi32(x2, 0xaa); + x3 = x1; + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x3 = _mm_bslli_si128(x3, 4); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 208), x1); + x2 = _mm_aeskeygenassist_si128(x1, 0x40); + x2 = _mm_shuffle_epi32(x2, 0xff); + x3 = x0; + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x3 = _mm_bslli_si128(x3, 4); + x0 = _mm_xor_si128(x0, x3); + x0 = _mm_xor_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 224), x0); +} + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_ECB_encrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10 = 0, r11 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x40; + r9 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_ECB_encrypt_AESNI_done_64; + } + r9 = (word32)((word32)r9 & 0xffffffc0); +L_AES_ECB_encrypt_AESNI_enc_64: + /* 64 bytes of input */ + /* aes_ecb_enc_64 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_xor_si128(x0, x4); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x4); + x3 = _mm_xor_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf3 = (word32)r8; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_ECB_encrypt_AESNI_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf5 = (word32)r8; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_ECB_encrypt_AESNI_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_encrypt_AESNI_64_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x4); + x1 = _mm_aesenclast_si128(x1, x4); + x2 = _mm_aesenclast_si128(x2, x4); + x3 = _mm_aesenclast_si128(x3, x4); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r11, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r11, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r11, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_AESNI_enc_64; + } +L_AES_ECB_encrypt_AESNI_done_64: + zf7 = (word32)rax; + zf8 = (word32)rdx; + r9 = (word32)((word32)rdx); + if ((zf7) == (zf8)) { + goto L_AES_ECB_encrypt_AESNI_done_enc; + } + r9 = (word32)((word32)r9 & 0xfffffff0); +L_AES_ECB_encrypt_AESNI_enc_16: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf9 = (word32)r8; + zf10 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_ECB_encrypt_AESNI_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf11 = (word32)r8; + zf12 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_ECB_encrypt_AESNI_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_encrypt_AESNI_16_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r10 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_AESNI_enc_16; + } +L_AES_ECB_encrypt_AESNI_done_enc: + ; +} + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_ECB_decrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10 = 0, r11 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x40; + r9 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_ECB_decrypt_AESNI_done_64; + } + r9 = (word32)((word32)r9 & 0xffffffc0); +L_AES_ECB_decrypt_AESNI_dec_64: + /* 64 bytes of input */ + /* aes_ecb_dec_64 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + /* aes_dec_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_xor_si128(x0, x4); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x4); + x3 = _mm_xor_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + zf3 = (word32)r8; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_ECB_decrypt_AESNI_64_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + zf5 = (word32)r8; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_ECB_decrypt_AESNI_64_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_decrypt_AESNI_64_aes_dec_block_last: + x0 = _mm_aesdeclast_si128(x0, x4); + x1 = _mm_aesdeclast_si128(x1, x4); + x2 = _mm_aesdeclast_si128(x2, x4); + x3 = _mm_aesdeclast_si128(x3, x4); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r11, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r11, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r11, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_AESNI_dec_64; + } +L_AES_ECB_decrypt_AESNI_done_64: + zf7 = (word32)rax; + zf8 = (word32)rdx; + r9 = (word32)((word32)rdx); + if ((zf7) == (zf8)) { + goto L_AES_ECB_decrypt_AESNI_done_dec; + } + r9 = (word32)((word32)r9 & 0xfffffff0); +L_AES_ECB_decrypt_AESNI_dec_16: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + /* aes_dec_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesdec_si128(x0, x5); + zf9 = (word32)r8; + zf10 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_ECB_decrypt_AESNI_16_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesdec_si128(x0, x6); + zf11 = (word32)r8; + zf12 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_ECB_decrypt_AESNI_16_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesdec_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_decrypt_AESNI_16_aes_dec_block_last: + x0 = _mm_aesdeclast_si128(x0, x5); + r10 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_AESNI_dec_16; + } +L_AES_ECB_decrypt_AESNI_done_dec: + ; +} + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_CBC_encrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10 = 0, r11 = 0; + __m128i x0, x1 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ivec; + rcx = (word64)(word64)length; + r8 = (word64)(size_t)KS; + r9 = (word64)(word32)nr; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + rax = (word32)(0); + if (((word32)rax) == ((word32)rcx)) { + goto L_AES_CBC_encrypt_AESNI_done; + } +L_AES_CBC_encrypt_AESNI_loop: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_xor_si128(x1, x0); + /* aes_enc_block */ + x1 = _mm_xor_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x1 = _mm_aesenc_si128(x1, x3); + zf1 = (word32)r9; + zf2 = 0xb; + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_CBC_encrypt_AESNI_aes_enc_block_last; + } + x1 = _mm_aesenc_si128(x1, x3); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x1 = _mm_aesenc_si128(x1, x4); + zf3 = (word32)r9; + zf4 = 0xd; + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CBC_encrypt_AESNI_aes_enc_block_last; + } + x1 = _mm_aesenc_si128(x1, x3); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x1 = _mm_aesenc_si128(x1, x4); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_encrypt_AESNI_aes_enc_block_last: + x1 = _mm_aesenclast_si128(x1, x3); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x1); + x0 = x1; + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)rcx)) { + goto L_AES_CBC_encrypt_AESNI_loop; + } +L_AES_CBC_encrypt_AESNI_done: + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); +} + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_CBC_decrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11 = 0, r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ivec; + rcx = (word64)(word64)length; + r8 = (word64)(size_t)KS; + r9 = (word64)(word32)nr; + + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + rax = (word32)(0); + zf1 = (word32)rcx; + zf2 = 0x40; + r10 = (word32)((word32)rcx); + if ((zf1) < (zf2)) { + goto L_AES_CBC_decrypt_AESNI_done_64; + } + r10 = (word32)((word32)r10 & 0xffffffc0); +L_AES_CBC_decrypt_AESNI_dec_64: + /* 64 bytes of input */ + /* aes_cbc_dec_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 48)); + /* aes_dec_block */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x0 = _mm_xor_si128(x0, x5); + x1 = _mm_xor_si128(x1, x5); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + zf3 = (word32)r9; + zf4 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CBC_decrypt_AESNI_64_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + zf5 = (word32)r9; + zf6 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_CBC_decrypt_AESNI_64_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_decrypt_AESNI_64_aes_dec_block_last: + x0 = _mm_aesdeclast_si128(x0, x5); + x1 = _mm_aesdeclast_si128(x1, x5); + x2 = _mm_aesdeclast_si128(x2, x5); + x3 = _mm_aesdeclast_si128(x3, x5); + x0 = _mm_xor_si128(x0, x4); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x1 = _mm_xor_si128(x1, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 16)); + x2 = _mm_xor_si128(x2, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 32)); + x3 = _mm_xor_si128(x3, x5); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 48)); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r12, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r12, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r12, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_AESNI_dec_64; + } +L_AES_CBC_decrypt_AESNI_done_64: + zf7 = (word32)rax; + zf8 = (word32)rcx; + r10 = (word32)((word32)rcx); + if ((zf7) == (zf8)) { + goto L_AES_CBC_decrypt_AESNI_done_dec; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_CBC_decrypt_AESNI_dec_16: + /* 16 bytes of input */ + r11 = (word64)(rdi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x8 = x0; + /* aes_dec_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x0 = _mm_aesdec_si128(x0, x6); + zf9 = (word32)r9; + zf10 = 0xb; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_CBC_decrypt_AESNI_16_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x6); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x0 = _mm_aesdec_si128(x0, x7); + zf11 = (word32)r9; + zf12 = 0xd; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_CBC_decrypt_AESNI_16_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x6); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x0 = _mm_aesdec_si128(x0, x7); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_decrypt_AESNI_16_aes_dec_block_last: + x0 = _mm_aesdeclast_si128(x0, x6); + x0 = _mm_xor_si128(x0, x4); + x4 = x8; + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_AESNI_dec_16; + } +L_AES_CBC_decrypt_AESNI_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x4); +} + +XALIGNED(16) static const word64 L_aes_ctr_aesni_bswap[] WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_aes_ctr_aesni_one[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("sse2,ssse3,sse4.1,aes") +WOLFSSL_LOCAL void AES_CTR_encrypt_AESNI(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11 = 0, r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7, x8, x9, x10, + x11 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + r9 = (word64)(size_t)ctr; + + x8 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_ctr_aesni_bswap, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_ctr_aesni_one, 0)); + x10 = _mm_setzero_si128(); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x7 = _mm_shuffle_epi8(x7, x8); + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x40; + r10 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_CTR_encrypt_AESNI_done_64; + } + r10 = (word32)((word32)r10 & 0xffffffc0); +L_AES_CTR_encrypt_AESNI_enc_64: + /* 64 bytes of input */ + /* aes_ctr_enc_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + x0 = x7; + x0 = _mm_shuffle_epi8(x0, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = x7; + x11 = _mm_cmpeq_epi64(x11, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + x1 = x7; + x1 = _mm_shuffle_epi8(x1, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = x7; + x11 = _mm_cmpeq_epi64(x11, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + x2 = x7; + x2 = _mm_shuffle_epi8(x2, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = x7; + x11 = _mm_cmpeq_epi64(x11, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + x3 = x7; + x3 = _mm_shuffle_epi8(x3, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = x7; + x11 = _mm_cmpeq_epi64(x11, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_xor_si128(x0, x4); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x4); + x3 = _mm_xor_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf3 = (word32)r8; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CTR_encrypt_AESNI_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf5 = (word32)r8; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_CTR_encrypt_AESNI_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_CTR_encrypt_AESNI_64_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x4); + x1 = _mm_aesenclast_si128(x1, x4); + x2 = _mm_aesenclast_si128(x2, x4); + x3 = _mm_aesenclast_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x0 = _mm_xor_si128(x0, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 16)); + x1 = _mm_xor_si128(x1, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 32)); + x2 = _mm_xor_si128(x2, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 48)); + x3 = _mm_xor_si128(x3, x4); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r12, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r12, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r12, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_AESNI_enc_64; + } +L_AES_CTR_encrypt_AESNI_done_64: + zf7 = (word32)rax; + zf8 = (word32)rdx; + r10 = (word32)((word32)rdx); + if ((zf7) == (zf8)) { + goto L_AES_CTR_encrypt_AESNI_done_enc; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_CTR_encrypt_AESNI_enc_16: + /* 16 bytes of input */ + x0 = x7; + x0 = _mm_shuffle_epi8(x0, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = x7; + x11 = _mm_cmpeq_epi64(x11, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf9 = (word32)r8; + zf10 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_CTR_encrypt_AESNI_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf11 = (word32)r8; + zf12 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_CTR_encrypt_AESNI_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_CTR_encrypt_AESNI_16_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r11 = (word64)(rdi + rax); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x0 = _mm_xor_si128(x0, x4); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_AESNI_enc_16; + } +L_AES_CTR_encrypt_AESNI_done_enc: + x7 = _mm_shuffle_epi8(x7, x8); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x7); +} + +#ifdef HAVE_INTEL_AVX1 +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_ECB_encrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10 = 0, r11 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x40; + r9 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_ECB_encrypt_avx1_done_64; + } + r9 = (word32)((word32)r9 & 0xffffffc0); +L_AES_ECB_encrypt_avx1_enc_64: + /* 64 bytes of input */ + /* aes_ecb_enc_64 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_xor_si128(x0, x4); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x4); + x3 = _mm_xor_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf3 = (word32)r8; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_ECB_encrypt_avx1_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf5 = (word32)r8; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_ECB_encrypt_avx1_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_encrypt_avx1_64_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x4); + x1 = _mm_aesenclast_si128(x1, x4); + x2 = _mm_aesenclast_si128(x2, x4); + x3 = _mm_aesenclast_si128(x3, x4); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r11, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r11, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r11, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_avx1_enc_64; + } +L_AES_ECB_encrypt_avx1_done_64: + zf7 = (word32)rax; + zf8 = (word32)rdx; + r9 = (word32)((word32)rdx); + if ((zf7) == (zf8)) { + goto L_AES_ECB_encrypt_avx1_done_enc; + } + r9 = (word32)((word32)r9 & 0xfffffff0); +L_AES_ECB_encrypt_avx1_enc_16: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf9 = (word32)r8; + zf10 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_ECB_encrypt_avx1_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf11 = (word32)r8; + zf12 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_ECB_encrypt_avx1_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_encrypt_avx1_16_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r10 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_avx1_enc_16; + } +L_AES_ECB_encrypt_avx1_done_enc: + ; +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_ECB_decrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10 = 0, r11 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x40; + r9 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_ECB_decrypt_avx1_done_64; + } + r9 = (word32)((word32)r9 & 0xffffffc0); +L_AES_ECB_decrypt_avx1_dec_64: + /* 64 bytes of input */ + /* aes_ecb_dec_64 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + /* aes_dec_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_xor_si128(x0, x4); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x4); + x3 = _mm_xor_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + zf3 = (word32)r8; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_ECB_decrypt_avx1_64_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + zf5 = (word32)r8; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_ECB_decrypt_avx1_64_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesdec_si128(x0, x4); + x1 = _mm_aesdec_si128(x1, x4); + x2 = _mm_aesdec_si128(x2, x4); + x3 = _mm_aesdec_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_decrypt_avx1_64_aes_dec_block_last: + x0 = _mm_aesdeclast_si128(x0, x4); + x1 = _mm_aesdeclast_si128(x1, x4); + x2 = _mm_aesdeclast_si128(x2, x4); + x3 = _mm_aesdeclast_si128(x3, x4); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r11, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r11, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r11, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_avx1_dec_64; + } +L_AES_ECB_decrypt_avx1_done_64: + zf7 = (word32)rax; + zf8 = (word32)rdx; + r9 = (word32)((word32)rdx); + if ((zf7) == (zf8)) { + goto L_AES_ECB_decrypt_avx1_done_dec; + } + r9 = (word32)((word32)r9 & 0xfffffff0); +L_AES_ECB_decrypt_avx1_dec_16: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + /* aes_dec_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesdec_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesdec_si128(x0, x5); + zf9 = (word32)r8; + zf10 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_ECB_decrypt_avx1_16_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesdec_si128(x0, x6); + zf11 = (word32)r8; + zf12 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_ECB_decrypt_avx1_16_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesdec_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_decrypt_avx1_16_aes_dec_block_last: + x0 = _mm_aesdeclast_si128(x0, x5); + r10 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_avx1_dec_16; + } +L_AES_ECB_decrypt_avx1_done_dec: + ; +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_CBC_encrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10 = 0, r11 = 0; + __m128i x0, x1 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ivec; + rcx = (word64)(word64)length; + r8 = (word64)(size_t)KS; + r9 = (word64)(word32)nr; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + rax = (word32)(0); + if (((word32)rax) == ((word32)rcx)) { + goto L_AES_CBC_encrypt_avx1_done; + } +L_AES_CBC_encrypt_avx1_loop: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_xor_si128(x1, x0); + /* aes_enc_block */ + x1 = _mm_xor_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x1 = _mm_aesenc_si128(x1, x3); + zf1 = (word32)r9; + zf2 = 0xb; + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_CBC_encrypt_avx1_aes_enc_block_last; + } + x1 = _mm_aesenc_si128(x1, x3); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x1 = _mm_aesenc_si128(x1, x4); + zf3 = (word32)r9; + zf4 = 0xd; + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CBC_encrypt_avx1_aes_enc_block_last; + } + x1 = _mm_aesenc_si128(x1, x3); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x1 = _mm_aesenc_si128(x1, x4); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_encrypt_avx1_aes_enc_block_last: + x1 = _mm_aesenclast_si128(x1, x3); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x1); + x0 = x1; + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)rcx)) { + goto L_AES_CBC_encrypt_avx1_loop; + } +L_AES_CBC_encrypt_avx1_done: + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_CBC_decrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11 = 0, r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), x4, + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ivec; + rcx = (word64)(word64)length; + r8 = (word64)(size_t)KS; + r9 = (word64)(word32)nr; + + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + rax = (word32)(0); + zf1 = (word32)rcx; + zf2 = 0x40; + r10 = (word32)((word32)rcx); + if ((zf1) < (zf2)) { + goto L_AES_CBC_decrypt_avx1_done_64; + } + r10 = (word32)((word32)r10 & 0xffffffc0); +L_AES_CBC_decrypt_avx1_dec_64: + /* 64 bytes of input */ + /* aes_cbc_dec_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 48)); + /* aes_dec_block */ + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x0 = _mm_xor_si128(x0, x5); + x1 = _mm_xor_si128(x1, x5); + x2 = _mm_xor_si128(x2, x5); + x3 = _mm_xor_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + zf3 = (word32)r9; + zf4 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CBC_decrypt_avx1_64_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + zf5 = (word32)r9; + zf6 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_CBC_decrypt_avx1_64_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x0 = _mm_aesdec_si128(x0, x5); + x1 = _mm_aesdec_si128(x1, x5); + x2 = _mm_aesdec_si128(x2, x5); + x3 = _mm_aesdec_si128(x3, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_decrypt_avx1_64_aes_dec_block_last: + x0 = _mm_aesdeclast_si128(x0, x5); + x1 = _mm_aesdeclast_si128(x1, x5); + x2 = _mm_aesdeclast_si128(x2, x5); + x3 = _mm_aesdeclast_si128(x3, x5); + x0 = _mm_xor_si128(x0, x4); + x1 = _mm_xor_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x2 = _mm_xor_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(r11, 16))); + x3 = _mm_xor_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(r11, 32))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 48)); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r12, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r12, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r12, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_avx1_dec_64; + } +L_AES_CBC_decrypt_avx1_done_64: + zf7 = (word32)rax; + zf8 = (word32)rcx; + r10 = (word32)((word32)rcx); + if ((zf7) == (zf8)) { + goto L_AES_CBC_decrypt_avx1_done_dec; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_CBC_decrypt_avx1_dec_16: + /* 16 bytes of input */ + r11 = (word64)(rdi + rax); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)); + x8 = x0; + /* aes_dec_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x0 = _mm_aesdec_si128(x0, x6); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x0 = _mm_aesdec_si128(x0, x6); + zf9 = (word32)r9; + zf10 = 0xb; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_CBC_decrypt_avx1_16_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x6); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x0 = _mm_aesdec_si128(x0, x7); + zf11 = (word32)r9; + zf12 = 0xd; + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_CBC_decrypt_avx1_16_aes_dec_block_last; + } + x0 = _mm_aesdec_si128(x0, x6); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x0 = _mm_aesdec_si128(x0, x7); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_decrypt_avx1_16_aes_dec_block_last: + x0 = _mm_aesdeclast_si128(x0, x6); + x0 = _mm_xor_si128(x0, x4); + x4 = x8; + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_avx1_dec_16; + } +L_AES_CBC_decrypt_avx1_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x4); +} + +XALIGNED(16) static const word64 L_aes_ctr_avx1_bswap[] WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(16) static const word64 L_aes_ctr_avx1_one[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_CTR_encrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11 = 0, r12 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7, x8, x9, x10, + x11 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + r9 = (word64)(size_t)ctr; + + x8 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_ctr_avx1_bswap, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_ctr_avx1_one, 0)); + x10 = _mm_setzero_si128(); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x7 = _mm_shuffle_epi8(x7, x8); + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x40; + r10 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_CTR_encrypt_avx1_done_64; + } + r10 = (word32)((word32)r10 & 0xffffffc0); +L_AES_CTR_encrypt_avx1_enc_64: + /* 64 bytes of input */ + /* aes_ctr_enc_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + x0 = _mm_shuffle_epi8(x7, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = _mm_cmpeq_epi64(x7, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + x1 = _mm_shuffle_epi8(x7, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = _mm_cmpeq_epi64(x7, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + x2 = _mm_shuffle_epi8(x7, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = _mm_cmpeq_epi64(x7, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + x3 = _mm_shuffle_epi8(x7, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = _mm_cmpeq_epi64(x7, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x0 = _mm_xor_si128(x0, x4); + x1 = _mm_xor_si128(x1, x4); + x2 = _mm_xor_si128(x2, x4); + x3 = _mm_xor_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf3 = (word32)r8; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CTR_encrypt_avx1_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + zf5 = (word32)r8; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_CTR_encrypt_avx1_64_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x4); + x1 = _mm_aesenc_si128(x1, x4); + x2 = _mm_aesenc_si128(x2, x4); + x3 = _mm_aesenc_si128(x3, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_CTR_encrypt_avx1_64_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x4); + x1 = _mm_aesenclast_si128(x1, x4); + x2 = _mm_aesenclast_si128(x2, x4); + x3 = _mm_aesenclast_si128(x3, x4); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x1 = _mm_xor_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(r11, 16))); + x2 = _mm_xor_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(r11, 32))); + x3 = _mm_xor_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(r11, 48))); + _mm_storeu_si128((__m128i*)WC_PW(r12, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r12, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r12, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r12, 48), x3); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_avx1_enc_64; + } +L_AES_CTR_encrypt_avx1_done_64: + zf7 = (word32)rax; + zf8 = (word32)rdx; + r10 = (word32)((word32)rdx); + if ((zf7) == (zf8)) { + goto L_AES_CTR_encrypt_avx1_done_enc; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_CTR_encrypt_avx1_enc_16: + /* 16 bytes of input */ + x0 = _mm_shuffle_epi8(x7, x8); + x7 = _mm_add_epi64(x7, x9); + x11 = _mm_cmpeq_epi64(x7, x10); + x11 = _mm_bslli_si128(x11, 8); + x11 = _mm_srli_epi64(x11, 63); + x7 = _mm_add_epi64(x7, x11); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf9 = (word32)r8; + zf10 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_CTR_encrypt_avx1_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf11 = (word32)r8; + zf12 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_CTR_encrypt_avx1_16_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_CTR_encrypt_avx1_16_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r11 = (word64)(rdi + rax); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x0); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_avx1_enc_16; + } +L_AES_CTR_encrypt_avx1_done_enc: + x7 = _mm_shuffle_epi8(x7, x8); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x7); +} + +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_VAES +WC_X64I_TARGET("aes,vaes,avx2") +WOLFSSL_LOCAL void AES_ECB_encrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10 = 0, r11 = 0; + __m128i x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(); + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x80; + r9 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_ECB_encrypt_vaes_done_128; + } + r9 = (word32)((word32)r9 & 0xffffff80); +L_AES_ECB_encrypt_vaes_enc_128: + /* 128 bytes of input */ + /* aes_ecb_enc_128 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + /* aes_enc_block */ + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y7); + y2 = _mm256_xor_si256(y2, y7); + y3 = _mm256_xor_si256(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + zf3 = (word32)r8; + zf4 = 0xb; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_ECB_encrypt_vaes_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + zf5 = (word32)r8; + zf6 = 0xd; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_ECB_encrypt_vaes_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y2 = _mm256_aesenc_epi128(y2, y7); + y3 = _mm256_aesenc_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_ECB_encrypt_vaes_128_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y7); + y1 = _mm256_aesenclast_epi128(y1, y7); + y2 = _mm256_aesenclast_epi128(y2, y7); + y3 = _mm256_aesenclast_epi128(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 96), y3); + rax = (word32)((word32)rax + 0x80); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_vaes_enc_128; + } +L_AES_ECB_encrypt_vaes_done_128: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 - (word32)rax); + if (((word32)r9) < (0x40)) { + goto L_AES_ECB_encrypt_vaes_done_64; + } + /* 64 bytes of input */ + /* aes_ecb_enc_64 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + /* aes_enc_block */ + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + zf7 = (word32)r8; + zf8 = 0xb; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_ECB_encrypt_vaes_64_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + zf9 = (word32)r8; + zf10 = 0xd; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_ECB_encrypt_vaes_64_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesenc_epi128(y0, y7); + y1 = _mm256_aesenc_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_ECB_encrypt_vaes_64_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y7); + y1 = _mm256_aesenclast_epi128(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 32), y1); + rax = (word32)((word32)rax + 0x40); +L_AES_ECB_encrypt_vaes_done_64: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 & 0xffffffe0); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_ECB_encrypt_vaes_done_32; + } +L_AES_ECB_encrypt_vaes_enc_32: + /* 32 bytes of input */ + /* aes_ecb_enc_32 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + /* aes_enc_block */ + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesenc_epi128(y0, y7); + zf11 = (word32)r8; + zf12 = 0xb; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_ECB_encrypt_vaes_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesenc_epi128(y0, y7); + zf13 = (word32)r8; + zf14 = 0xd; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_ECB_encrypt_vaes_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesenc_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_ECB_encrypt_vaes_32_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), y0); + rax = (word32)((word32)rax + 0x20); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_vaes_enc_32; + } +L_AES_ECB_encrypt_vaes_done_32: + zf15 = (word32)rax; + zf16 = (word32)rdx; + r9 = (word32)((word32)rdx); + if ((zf15) == (zf16)) { + goto L_AES_ECB_encrypt_vaes_done_enc; + } + r9 = (word32)((word32)r9 & 0xfffffff0); +L_AES_ECB_encrypt_vaes_enc_16: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + /* aes_enc_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + zf17 = (word32)r8; + zf18 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_ECB_encrypt_vaes_16_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x6)); + zf19 = (word32)r8; + zf20 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_ECB_encrypt_vaes_16_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + x6)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_encrypt_vaes_16_aes_enc_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y0), + x5)); + r10 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), _mm256_castsi256_si128(y0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_vaes_enc_16; + } +L_AES_ECB_encrypt_vaes_done_enc: + ; +} + +WC_X64I_TARGET("aes,vaes,avx2") +WOLFSSL_LOCAL void AES_ECB_decrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10 = 0, r11 = 0; + __m128i x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(); + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x80; + r9 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_ECB_decrypt_vaes_done_128; + } + r9 = (word32)((word32)r9 & 0xffffff80); +L_AES_ECB_decrypt_vaes_dec_128: + /* 128 bytes of input */ + /* aes_ecb_dec_128 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + /* aes_dec_block */ + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y7); + y2 = _mm256_xor_si256(y2, y7); + y3 = _mm256_xor_si256(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + zf3 = (word32)r8; + zf4 = 0xb; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_ECB_decrypt_vaes_128_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + zf5 = (word32)r8; + zf6 = 0xd; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_ECB_decrypt_vaes_128_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y2 = _mm256_aesdec_epi128(y2, y7); + y3 = _mm256_aesdec_epi128(y3, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_ECB_decrypt_vaes_128_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y7); + y1 = _mm256_aesdeclast_epi128(y1, y7); + y2 = _mm256_aesdeclast_epi128(y2, y7); + y3 = _mm256_aesdeclast_epi128(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 96), y3); + rax = (word32)((word32)rax + 0x80); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_vaes_dec_128; + } +L_AES_ECB_decrypt_vaes_done_128: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 - (word32)rax); + if (((word32)r9) < (0x40)) { + goto L_AES_ECB_decrypt_vaes_done_64; + } + /* 64 bytes of input */ + /* aes_ecb_dec_64 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + /* aes_dec_block */ + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + zf7 = (word32)r8; + zf8 = 0xb; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_ECB_decrypt_vaes_64_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + zf9 = (word32)r8; + zf10 = 0xd; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_ECB_decrypt_vaes_64_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesdec_epi128(y0, y7); + y1 = _mm256_aesdec_epi128(y1, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_ECB_decrypt_vaes_64_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y7); + y1 = _mm256_aesdeclast_epi128(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 32), y1); + rax = (word32)((word32)rax + 0x40); +L_AES_ECB_decrypt_vaes_done_64: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 & 0xffffffe0); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_ECB_decrypt_vaes_done_32; + } +L_AES_ECB_decrypt_vaes_dec_32: + /* 32 bytes of input */ + /* aes_ecb_dec_32 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + /* aes_dec_block */ + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesdec_epi128(y0, y7); + zf11 = (word32)r8; + zf12 = 0xb; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_ECB_decrypt_vaes_32_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesdec_epi128(y0, y7); + zf13 = (word32)r8; + zf14 = 0xd; + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_ECB_decrypt_vaes_32_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesdec_epi128(y0, y7); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_ECB_decrypt_vaes_32_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), y0); + rax = (word32)((word32)rax + 0x20); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_vaes_dec_32; + } +L_AES_ECB_decrypt_vaes_done_32: + zf15 = (word32)rax; + zf16 = (word32)rdx; + r9 = (word32)((word32)rdx); + if ((zf15) == (zf16)) { + goto L_AES_ECB_decrypt_vaes_done_dec; + } + r9 = (word32)((word32)r9 & 0xfffffff0); +L_AES_ECB_decrypt_vaes_dec_16: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + /* aes_dec_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + zf17 = (word32)r8; + zf18 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_ECB_decrypt_vaes_16_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x6)); + zf19 = (word32)r8; + zf20 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_ECB_decrypt_vaes_16_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x6)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_decrypt_vaes_16_aes_dec_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesdeclast_si128(_mm256_castsi256_si128(y0), + x5)); + r10 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), _mm256_castsi256_si128(y0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_vaes_dec_16; + } +L_AES_ECB_decrypt_vaes_done_dec: + ; +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_CBC_encrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10 = 0, r11 = 0; + __m128i x0, x1 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ivec; + rcx = (word64)(word64)length; + r8 = (word64)(size_t)KS; + r9 = (word64)(word32)nr; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + rax = (word32)(0); + if (((word32)rax) == ((word32)rcx)) { + goto L_AES_CBC_encrypt_vaes_done; + } +L_AES_CBC_encrypt_vaes_loop: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_xor_si128(x1, x0); + /* aes_enc_block */ + x1 = _mm_xor_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x1 = _mm_aesenc_si128(x1, x3); + zf1 = (word32)r9; + zf2 = 0xb; + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_CBC_encrypt_vaes_aes_enc_block_last; + } + x1 = _mm_aesenc_si128(x1, x3); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x1 = _mm_aesenc_si128(x1, x4); + zf3 = (word32)r9; + zf4 = 0xd; + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CBC_encrypt_vaes_aes_enc_block_last; + } + x1 = _mm_aesenc_si128(x1, x3); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x1 = _mm_aesenc_si128(x1, x4); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_encrypt_vaes_aes_enc_block_last: + x1 = _mm_aesenclast_si128(x1, x3); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x1); + x0 = x1; + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)rcx)) { + goto L_AES_CBC_encrypt_vaes_loop; + } +L_AES_CBC_encrypt_vaes_done: + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); +} + +WC_X64I_TARGET("aes,vaes,avx2") +WOLFSSL_LOCAL void AES_CBC_decrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11 = 0, r12 = 0; + __m128i x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(); + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), y8, + y9 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ivec; + rcx = (word64)(word64)length; + r8 = (word64)(size_t)KS; + r9 = (word64)(word32)nr; + + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + rax = (word32)(0); + zf1 = (word32)rcx; + zf2 = 0x80; + r10 = (word32)((word32)rcx); + if ((zf1) < (zf2)) { + goto L_AES_CBC_decrypt_vaes_done_128; + } + r10 = (word32)((word32)r10 & 0xffffff80); +L_AES_CBC_decrypt_vaes_dec_128: + /* 128 bytes of input */ + /* aes_cbc_dec_128 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 96)); + y10 = _mm256_inserti128_si256(y8, _mm256_castsi256_si128(y0), 1); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 16)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 48)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 80)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + /* aes_dec_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 0))); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y9); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + zf3 = (word32)r9; + zf4 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CBC_decrypt_vaes_128_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + zf5 = (word32)r9; + zf6 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_CBC_decrypt_vaes_128_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_CBC_decrypt_vaes_128_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y9); + y1 = _mm256_aesdeclast_epi128(y1, y9); + y2 = _mm256_aesdeclast_epi128(y2, y9); + y3 = _mm256_aesdeclast_epi128(y3, y9); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 96), y3); + rax = (word32)((word32)rax + 0x80); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_vaes_dec_128; + } +L_AES_CBC_decrypt_vaes_done_128: + r10 = (word32)((word32)rcx); + r10 = (word32)((word32)r10 - (word32)rax); + if (((word32)r10) < (0x40)) { + goto L_AES_CBC_decrypt_vaes_done_64; + } + /* 64 bytes of input */ + /* aes_cbc_dec_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 32)); + y10 = _mm256_inserti128_si256(y8, _mm256_castsi256_si128(y0), 1); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 16)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + /* aes_dec_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 0))); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + zf7 = (word32)r9; + zf8 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_CBC_decrypt_vaes_64_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + zf9 = (word32)r9; + zf10 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_CBC_decrypt_vaes_64_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_CBC_decrypt_vaes_64_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y9); + y1 = _mm256_aesdeclast_epi128(y1, y9); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 32), y1); + rax = (word32)((word32)rax + 0x40); +L_AES_CBC_decrypt_vaes_done_64: + r10 = (word32)((word32)rcx); + r10 = (word32)((word32)r10 & 0xffffffe0); + if (((word32)rax) == ((word32)r10)) { + goto L_AES_CBC_decrypt_vaes_done_32; + } +L_AES_CBC_decrypt_vaes_dec_32: + /* 32 bytes of input */ + /* aes_cbc_dec_32 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r11, 0)); + y10 = _mm256_inserti128_si256(y8, _mm256_castsi256_si128(y0), 1); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + /* aes_dec_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 0))); + y0 = _mm256_xor_si256(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_aesdec_epi128(y0, y9); + zf11 = (word32)r9; + zf12 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_CBC_decrypt_vaes_32_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_aesdec_epi128(y0, y9); + zf13 = (word32)r9; + zf14 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_CBC_decrypt_vaes_32_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_CBC_decrypt_vaes_32_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y9); + y0 = _mm256_xor_si256(y0, y10); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y0); + rax = (word32)((word32)rax + 0x20); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_vaes_dec_32; + } +L_AES_CBC_decrypt_vaes_done_32: + zf15 = (word32)rax; + zf16 = (word32)rcx; + r10 = (word32)((word32)rcx); + if ((zf15) == (zf16)) { + goto L_AES_CBC_decrypt_vaes_done_dec; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_CBC_decrypt_vaes_dec_16: + /* 16 bytes of input */ + r11 = (word64)(rdi + rax); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x7 = _mm256_castsi256_si128(y0); + /* aes_dec_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + zf17 = (word32)r9; + zf18 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_CBC_decrypt_vaes_16_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x6)); + zf19 = (word32)r9; + zf20 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_CBC_decrypt_vaes_16_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + x6)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_decrypt_vaes_16_aes_dec_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesdeclast_si128(_mm256_castsi256_si128(y0), + x5)); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(x7); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), _mm256_castsi256_si128(y0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_vaes_dec_16; + } +L_AES_CBC_decrypt_vaes_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y8)); +} + +XALIGNED(16) static const word64 L_aes_ctr_bswap_vaes[] WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(32) static const word64 L_aes_ctr_inc_vaes[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0x0000000000000002ULL, 0x0000000000000000ULL, + 0x0000000000000003ULL, 0x0000000000000000ULL, + 0x0000000000000004ULL, 0x0000000000000000ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000000000008ULL, 0x0000000000000000ULL, + 0x0000000000000009ULL, 0x0000000000000000ULL, + 0x000000000000000aULL, 0x0000000000000000ULL, + 0x000000000000000bULL, 0x0000000000000000ULL, + 0x000000000000000cULL, 0x0000000000000000ULL, + 0x000000000000000dULL, 0x0000000000000000ULL, + 0x000000000000000eULL, 0x0000000000000000ULL, + 0x000000000000000fULL, 0x0000000000000000ULL, + 0x0000000000000010ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("aes,vaes,avx2") +WOLFSSL_LOCAL void AES_CTR_encrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11 = 0, r12 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7, y8, y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), y12, + y13, y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + r9 = (word64)(size_t)ctr; + + y8 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_bswap_vaes, 0))); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 0))); + y7 = _mm256_shuffle_epi8(y7, y8); + y12 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_inc_vaes, 32))); + y13 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_inc_vaes, 16))); + rax = (word32)(0); + zf1 = (word32)rdx; + zf2 = 0x80; + r10 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_CTR_encrypt_vaes_done_128; + } + r10 = (word32)((word32)r10 & 0xffffff80); + y10 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_inc_vaes, 128))); + y9 = y7; + y4 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 0))); + y15 = _mm256_and_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 0))); + y9 = _mm256_or_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 0))); + y9 = _mm256_andnot_si256(y4, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y4 = _mm256_add_epi64(y4, y9); + y9 = y7; + y5 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 32))); + y15 = _mm256_and_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 32))); + y9 = _mm256_or_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 32))); + y9 = _mm256_andnot_si256(y5, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_add_epi64(y5, y9); + y9 = y7; + y6 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 64))); + y15 = _mm256_and_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 64))); + y9 = _mm256_or_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 64))); + y9 = _mm256_andnot_si256(y6, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_add_epi64(y6, y9); + y9 = y7; + y7 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 96))); + y15 = _mm256_and_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 96))); + y9 = _mm256_or_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 96))); + y9 = _mm256_andnot_si256(y7, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_add_epi64(y7, y9); +L_AES_CTR_encrypt_vaes_enc_128: + /* 128 bytes of input */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + y0 = _mm256_shuffle_epi8(y4, y8); + y1 = _mm256_shuffle_epi8(y5, y8); + y2 = _mm256_shuffle_epi8(y6, y8); + y3 = _mm256_shuffle_epi8(y7, y8); + y9 = y4; + y4 = _mm256_add_epi64(y4, y10); + y15 = _mm256_and_si256(y9, y10); + y9 = _mm256_or_si256(y9, y10); + y9 = _mm256_andnot_si256(y4, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y4 = _mm256_add_epi64(y4, y9); + y9 = y5; + y5 = _mm256_add_epi64(y5, y10); + y15 = _mm256_and_si256(y9, y10); + y9 = _mm256_or_si256(y9, y10); + y9 = _mm256_andnot_si256(y5, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_add_epi64(y5, y9); + y9 = y6; + y6 = _mm256_add_epi64(y6, y10); + y15 = _mm256_and_si256(y9, y10); + y9 = _mm256_or_si256(y9, y10); + y9 = _mm256_andnot_si256(y6, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_add_epi64(y6, y9); + y9 = y7; + y7 = _mm256_add_epi64(y7, y10); + y15 = _mm256_and_si256(y9, y10); + y9 = _mm256_or_si256(y9, y10); + y9 = _mm256_andnot_si256(y7, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_add_epi64(y7, y9); + /* aes_enc_block */ + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y14); + y1 = _mm256_xor_si256(y1, y14); + y2 = _mm256_xor_si256(y2, y14); + y3 = _mm256_xor_si256(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + zf3 = (word32)r8; + zf4 = 0xb; + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CTR_encrypt_vaes_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + zf5 = (word32)r8; + zf6 = 0xd; + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_CTR_encrypt_vaes_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y2 = _mm256_aesenc_epi128(y2, y14); + y3 = _mm256_aesenc_epi128(y3, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_CTR_encrypt_vaes_128_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y14); + y1 = _mm256_aesenclast_epi128(y1, y14); + y2 = _mm256_aesenclast_epi128(y2, y14); + y3 = _mm256_aesenclast_epi128(y3, y14); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32))); + y2 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64))); + y3 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 96), y3); + rax = (word32)((word32)rax + 0x80); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_vaes_enc_128; + } + y7 = _mm256_permute2x128_si256(y4, y4, 0); +L_AES_CTR_encrypt_vaes_done_128: + r10 = (word32)((word32)rdx); + r10 = (word32)((word32)r10 - (word32)rax); + if (((word32)r10) < (0x40)) { + goto L_AES_CTR_encrypt_vaes_done_64; + } + /* 64 bytes of input */ + y11 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_inc_vaes, 64))); + /* aes_ctr_enc_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + y0 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 0))); + y9 = y7; + y15 = _mm256_and_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 0))); + y9 = _mm256_or_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 0))); + y9 = _mm256_andnot_si256(y0, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y0 = _mm256_add_epi64(y0, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 32))); + y9 = y7; + y15 = _mm256_and_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 32))); + y9 = _mm256_or_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 32))); + y9 = _mm256_andnot_si256(y1, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y1 = _mm256_add_epi64(y1, y9); + y1 = _mm256_shuffle_epi8(y1, y8); + y9 = y7; + y7 = _mm256_add_epi64(y7, y11); + y15 = _mm256_and_si256(y9, y11); + y9 = _mm256_or_si256(y9, y11); + y9 = _mm256_andnot_si256(y7, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_add_epi64(y7, y9); + /* aes_enc_block */ + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y14); + y1 = _mm256_xor_si256(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + zf7 = (word32)r8; + zf8 = 0xb; + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_CTR_encrypt_vaes_64_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + zf9 = (word32)r8; + zf10 = 0xd; + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_CTR_encrypt_vaes_64_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesenc_epi128(y0, y14); + y1 = _mm256_aesenc_epi128(y1, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_CTR_encrypt_vaes_64_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y14); + y1 = _mm256_aesenclast_epi128(y1, y14); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 32), y1); + rax = (word32)((word32)rax + 0x40); +L_AES_CTR_encrypt_vaes_done_64: + r10 = (word32)((word32)rdx); + r10 = (word32)((word32)r10 & 0xffffffe0); + if (((word32)rax) == ((word32)r10)) { + goto L_AES_CTR_encrypt_vaes_done_32; + } +L_AES_CTR_encrypt_vaes_enc_32: + /* 32 bytes of input */ + /* aes_ctr_enc_32 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + y0 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 0))); + y9 = y7; + y15 = _mm256_and_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 0))); + y9 = _mm256_or_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR( + L_aes_ctr_inc_vaes, 0))); + y9 = _mm256_andnot_si256(y0, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y0 = _mm256_add_epi64(y0, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y9 = y7; + y7 = _mm256_add_epi64(y7, y12); + y15 = _mm256_and_si256(y9, y12); + y9 = _mm256_or_si256(y9, y12); + y9 = _mm256_andnot_si256(y7, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_add_epi64(y7, y9); + /* aes_enc_block */ + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + y0 = _mm256_xor_si256(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_aesenc_epi128(y0, y14); + zf11 = (word32)r8; + zf12 = 0xb; + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_CTR_encrypt_vaes_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_aesenc_epi128(y0, y14); + zf13 = (word32)r8; + zf14 = 0xd; + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_CTR_encrypt_vaes_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_aesenc_epi128(y0, y14); + y14 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_CTR_encrypt_vaes_32_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y14); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y0); + rax = (word32)((word32)rax + 0x20); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_vaes_enc_32; + } +L_AES_CTR_encrypt_vaes_done_32: + zf15 = (word32)rax; + zf16 = (word32)rdx; + r10 = (word32)((word32)rdx); + if ((zf15) == (zf16)) { + goto L_AES_CTR_encrypt_vaes_done_enc; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_CTR_encrypt_vaes_enc_16: + /* 16 bytes of input */ + y0 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y9 = y7; + y7 = _mm256_add_epi64(y7, y13); + y15 = _mm256_and_si256(y9, y13); + y9 = _mm256_or_si256(y9, y13); + y9 = _mm256_andnot_si256(y7, y9); + y9 = _mm256_or_si256(y9, y15); + y9 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_add_epi64(y7, y9); + /* aes_enc_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf17 = (word32)r8; + zf18 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_CTR_encrypt_vaes_16_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf19 = (word32)r8; + zf20 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_CTR_encrypt_vaes_16_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_CTR_encrypt_vaes_16_aes_enc_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + r11 = (word64)(rdi + rax); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)))); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), _mm256_castsi256_si128(y0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_vaes_enc_16; + } +L_AES_CTR_encrypt_vaes_done_enc: + y0 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm256_castsi256_si128(y0)); +} + +#endif /* HAVE_INTEL_VAES */ +#ifdef HAVE_INTEL_AVX512 +WC_X64I_TARGET("aes,vaes,avx512f,avx512vl") +WOLFSSL_LOCAL void AES_ECB_encrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9 = 0, r10 = 0, r11 = 0; + __m128i x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(); + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z7 = _mm512_setzero_si512(), z8 = _mm512_setzero_si512(), + z9 = _mm512_setzero_si512(), z10 = _mm512_setzero_si512(), + z11 = _mm512_setzero_si512(), z12 = _mm512_setzero_si512(), + z13 = _mm512_setzero_si512(), z14 = _mm512_setzero_si512(), + z15 = _mm512_setzero_si512(), z16 = _mm512_setzero_si512(), + z17 = _mm512_setzero_si512(), z18 = _mm512_setzero_si512(), + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + + rax = (word32)(0); + if (((word32)rdx) < (0x20)) { + goto L_AES_ECB_encrypt_avx512_done_32; + } + z8 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + z16 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + z17 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + z18 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)((word32)r8) < (sword32)(0xb)) { + goto L_AES_ECB_encrypt_avx512_key_cached; + } + z19 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)((word32)r8) < (sword32)(0xd)) { + goto L_AES_ECB_encrypt_avx512_key_cached; + } + z21 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_ECB_encrypt_avx512_key_cached: + zf1 = (word32)rdx; + zf2 = 0x100; + r9 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_ECB_encrypt_avx512_done_256; + } + r9 = (word32)((word32)r9 & 0xffffff00); +L_AES_ECB_encrypt_avx512_enc_256: + /* 256 bytes of input */ + /* aes_ecb_enc_256 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r10, 192)); + /* aes_enc_block */ + z0 = _mm512_xor_si512(z0, z8); + z1 = _mm512_xor_si512(z1, z8); + z2 = _mm512_xor_si512(z2, z8); + z3 = _mm512_xor_si512(z3, z8); + z0 = _mm512_aesenc_epi128(z0, z9); + z1 = _mm512_aesenc_epi128(z1, z9); + z2 = _mm512_aesenc_epi128(z2, z9); + z3 = _mm512_aesenc_epi128(z3, z9); + z0 = _mm512_aesenc_epi128(z0, z10); + z1 = _mm512_aesenc_epi128(z1, z10); + z2 = _mm512_aesenc_epi128(z2, z10); + z3 = _mm512_aesenc_epi128(z3, z10); + z0 = _mm512_aesenc_epi128(z0, z11); + z1 = _mm512_aesenc_epi128(z1, z11); + z2 = _mm512_aesenc_epi128(z2, z11); + z3 = _mm512_aesenc_epi128(z3, z11); + z0 = _mm512_aesenc_epi128(z0, z12); + z1 = _mm512_aesenc_epi128(z1, z12); + z2 = _mm512_aesenc_epi128(z2, z12); + z3 = _mm512_aesenc_epi128(z3, z12); + z0 = _mm512_aesenc_epi128(z0, z13); + z1 = _mm512_aesenc_epi128(z1, z13); + z2 = _mm512_aesenc_epi128(z2, z13); + z3 = _mm512_aesenc_epi128(z3, z13); + z0 = _mm512_aesenc_epi128(z0, z14); + z1 = _mm512_aesenc_epi128(z1, z14); + z2 = _mm512_aesenc_epi128(z2, z14); + z3 = _mm512_aesenc_epi128(z3, z14); + z0 = _mm512_aesenc_epi128(z0, z15); + z1 = _mm512_aesenc_epi128(z1, z15); + z2 = _mm512_aesenc_epi128(z2, z15); + z3 = _mm512_aesenc_epi128(z3, z15); + z0 = _mm512_aesenc_epi128(z0, z16); + z1 = _mm512_aesenc_epi128(z1, z16); + z2 = _mm512_aesenc_epi128(z2, z16); + z3 = _mm512_aesenc_epi128(z3, z16); + z0 = _mm512_aesenc_epi128(z0, z17); + z1 = _mm512_aesenc_epi128(z1, z17); + z2 = _mm512_aesenc_epi128(z2, z17); + z3 = _mm512_aesenc_epi128(z3, z17); + zf3 = (word32)r8; + zf4 = 0xb; + z7 = z18; + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_ECB_encrypt_avx512_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z18); + z1 = _mm512_aesenc_epi128(z1, z18); + z2 = _mm512_aesenc_epi128(z2, z18); + z3 = _mm512_aesenc_epi128(z3, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z1 = _mm512_aesenc_epi128(z1, z19); + z2 = _mm512_aesenc_epi128(z2, z19); + z3 = _mm512_aesenc_epi128(z3, z19); + zf5 = (word32)r8; + zf6 = 0xd; + z7 = z20; + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_ECB_encrypt_avx512_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z20); + z1 = _mm512_aesenc_epi128(z1, z20); + z2 = _mm512_aesenc_epi128(z2, z20); + z3 = _mm512_aesenc_epi128(z3, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z1 = _mm512_aesenc_epi128(z1, z21); + z2 = _mm512_aesenc_epi128(z2, z21); + z3 = _mm512_aesenc_epi128(z3, z21); + z7 = z22; +L_AES_ECB_encrypt_avx512_256_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z7); + z1 = _mm512_aesenclast_epi128(z1, z7); + z2 = _mm512_aesenclast_epi128(z2, z7); + z3 = _mm512_aesenclast_epi128(z3, z7); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z1); + _mm512_storeu_si512((void*)WC_PW(r11, 128), z2); + _mm512_storeu_si512((void*)WC_PW(r11, 192), z3); + rax = (word32)((word32)rax + 0x100); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_avx512_enc_256; + } +L_AES_ECB_encrypt_avx512_done_256: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 - (word32)rax); + if (((word32)r9) < (0x80)) { + goto L_AES_ECB_encrypt_avx512_done_128; + } + /* 128 bytes of input */ + /* aes_ecb_enc_128 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + /* aes_enc_block */ + z0 = _mm512_xor_si512(z0, z8); + z1 = _mm512_xor_si512(z1, z8); + z0 = _mm512_aesenc_epi128(z0, z9); + z1 = _mm512_aesenc_epi128(z1, z9); + z0 = _mm512_aesenc_epi128(z0, z10); + z1 = _mm512_aesenc_epi128(z1, z10); + z0 = _mm512_aesenc_epi128(z0, z11); + z1 = _mm512_aesenc_epi128(z1, z11); + z0 = _mm512_aesenc_epi128(z0, z12); + z1 = _mm512_aesenc_epi128(z1, z12); + z0 = _mm512_aesenc_epi128(z0, z13); + z1 = _mm512_aesenc_epi128(z1, z13); + z0 = _mm512_aesenc_epi128(z0, z14); + z1 = _mm512_aesenc_epi128(z1, z14); + z0 = _mm512_aesenc_epi128(z0, z15); + z1 = _mm512_aesenc_epi128(z1, z15); + z0 = _mm512_aesenc_epi128(z0, z16); + z1 = _mm512_aesenc_epi128(z1, z16); + z0 = _mm512_aesenc_epi128(z0, z17); + z1 = _mm512_aesenc_epi128(z1, z17); + zf7 = (word32)r8; + zf8 = 0xb; + z7 = z18; + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_ECB_encrypt_avx512_128_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z18); + z1 = _mm512_aesenc_epi128(z1, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z1 = _mm512_aesenc_epi128(z1, z19); + zf9 = (word32)r8; + zf10 = 0xd; + z7 = z20; + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_ECB_encrypt_avx512_128_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z20); + z1 = _mm512_aesenc_epi128(z1, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z1 = _mm512_aesenc_epi128(z1, z21); + z7 = z22; +L_AES_ECB_encrypt_avx512_128_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z7); + z1 = _mm512_aesenclast_epi128(z1, z7); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z1); + rax = (word32)((word32)rax + 0x80); +L_AES_ECB_encrypt_avx512_done_128: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 & 0xffffffc0); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_ECB_encrypt_avx512_done_64; + } +L_AES_ECB_encrypt_avx512_enc_64: + /* 64 bytes of input */ + /* aes_ecb_enc_64 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + /* aes_enc_block */ + z0 = _mm512_xor_si512(z0, z8); + z0 = _mm512_aesenc_epi128(z0, z9); + z0 = _mm512_aesenc_epi128(z0, z10); + z0 = _mm512_aesenc_epi128(z0, z11); + z0 = _mm512_aesenc_epi128(z0, z12); + z0 = _mm512_aesenc_epi128(z0, z13); + z0 = _mm512_aesenc_epi128(z0, z14); + z0 = _mm512_aesenc_epi128(z0, z15); + z0 = _mm512_aesenc_epi128(z0, z16); + z0 = _mm512_aesenc_epi128(z0, z17); + zf11 = (word32)r8; + zf12 = 0xb; + z7 = z18; + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_ECB_encrypt_avx512_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + zf13 = (word32)r8; + zf14 = 0xd; + z7 = z20; + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_ECB_encrypt_avx512_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z7 = z22; +L_AES_ECB_encrypt_avx512_64_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z7); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z0); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_avx512_enc_64; + } +L_AES_ECB_encrypt_avx512_done_64: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 - (word32)rax); + if (((word32)r9) < (0x20)) { + goto L_AES_ECB_encrypt_avx512_done_32; + } + /* 32 bytes of input */ + r10 = (word64)(rdi + rax); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0))); + /* aes_enc_block */ + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z8))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z9))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z10))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z11))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z12))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z13))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z14))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z15))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z16))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z17))); + zf15 = (word32)r8; + zf16 = 0xb; + z7 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z18)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_ECB_encrypt_avx512_32_aes_enc_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z18))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z19))); + zf17 = (word32)r8; + zf18 = 0xd; + z7 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z20)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_ECB_encrypt_avx512_32_aes_enc_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z20))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z21))); + z7 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z22)); +L_AES_ECB_encrypt_avx512_32_aes_enc_block_last: + z0 = _mm512_zextsi256_si512(_mm256_aesenclast_epi128(_mm512_castsi512_si256( + z0), _mm512_castsi512_si256(z7))); + r10 = (word64)(rsi + rax); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), _mm512_castsi512_si256(z0)); + rax = (word32)((word32)rax + 0x20); +L_AES_ECB_encrypt_avx512_done_32: + zf19 = (word32)rax; + zf20 = (word32)rdx; + r9 = (word32)((word32)rdx); + if ((zf19) == (zf20)) { + goto L_AES_ECB_encrypt_avx512_done_enc; + } + r9 = (word32)((word32)r9 & 0xfffffff0); +L_AES_ECB_encrypt_avx512_enc_16: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + /* aes_enc_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + zf21 = (word32)r8; + zf22 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_ECB_encrypt_avx512_16_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x6)); + zf23 = (word32)r8; + zf24 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_ECB_encrypt_avx512_16_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + x6)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_encrypt_avx512_16_aes_enc_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z0), + x5)); + r10 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), _mm512_castsi512_si128(z0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_encrypt_avx512_enc_16; + } +L_AES_ECB_encrypt_avx512_done_enc: + ; +} + +WC_X64I_TARGET("aes,vaes,avx512f,avx512vl") +WOLFSSL_LOCAL void AES_ECB_decrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9 = 0, r10 = 0, r11 = 0; + __m128i x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(); + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z7 = _mm512_setzero_si512(), z8 = _mm512_setzero_si512(), + z9 = _mm512_setzero_si512(), z10 = _mm512_setzero_si512(), + z11 = _mm512_setzero_si512(), z12 = _mm512_setzero_si512(), + z13 = _mm512_setzero_si512(), z14 = _mm512_setzero_si512(), + z15 = _mm512_setzero_si512(), z16 = _mm512_setzero_si512(), + z17 = _mm512_setzero_si512(), z18 = _mm512_setzero_si512(), + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + + rax = (word32)(0); + if (((word32)rdx) < (0x20)) { + goto L_AES_ECB_decrypt_avx512_done_32; + } + z8 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + z16 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + z17 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + z18 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)((word32)r8) < (sword32)(0xb)) { + goto L_AES_ECB_decrypt_avx512_key_cached; + } + z19 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)((word32)r8) < (sword32)(0xd)) { + goto L_AES_ECB_decrypt_avx512_key_cached; + } + z21 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_ECB_decrypt_avx512_key_cached: + zf1 = (word32)rdx; + zf2 = 0x100; + r9 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_ECB_decrypt_avx512_done_256; + } + r9 = (word32)((word32)r9 & 0xffffff00); +L_AES_ECB_decrypt_avx512_dec_256: + /* 256 bytes of input */ + /* aes_ecb_dec_256 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r10, 192)); + /* aes_dec_block */ + z0 = _mm512_xor_si512(z0, z8); + z1 = _mm512_xor_si512(z1, z8); + z2 = _mm512_xor_si512(z2, z8); + z3 = _mm512_xor_si512(z3, z8); + z0 = _mm512_aesdec_epi128(z0, z9); + z1 = _mm512_aesdec_epi128(z1, z9); + z2 = _mm512_aesdec_epi128(z2, z9); + z3 = _mm512_aesdec_epi128(z3, z9); + z0 = _mm512_aesdec_epi128(z0, z10); + z1 = _mm512_aesdec_epi128(z1, z10); + z2 = _mm512_aesdec_epi128(z2, z10); + z3 = _mm512_aesdec_epi128(z3, z10); + z0 = _mm512_aesdec_epi128(z0, z11); + z1 = _mm512_aesdec_epi128(z1, z11); + z2 = _mm512_aesdec_epi128(z2, z11); + z3 = _mm512_aesdec_epi128(z3, z11); + z0 = _mm512_aesdec_epi128(z0, z12); + z1 = _mm512_aesdec_epi128(z1, z12); + z2 = _mm512_aesdec_epi128(z2, z12); + z3 = _mm512_aesdec_epi128(z3, z12); + z0 = _mm512_aesdec_epi128(z0, z13); + z1 = _mm512_aesdec_epi128(z1, z13); + z2 = _mm512_aesdec_epi128(z2, z13); + z3 = _mm512_aesdec_epi128(z3, z13); + z0 = _mm512_aesdec_epi128(z0, z14); + z1 = _mm512_aesdec_epi128(z1, z14); + z2 = _mm512_aesdec_epi128(z2, z14); + z3 = _mm512_aesdec_epi128(z3, z14); + z0 = _mm512_aesdec_epi128(z0, z15); + z1 = _mm512_aesdec_epi128(z1, z15); + z2 = _mm512_aesdec_epi128(z2, z15); + z3 = _mm512_aesdec_epi128(z3, z15); + z0 = _mm512_aesdec_epi128(z0, z16); + z1 = _mm512_aesdec_epi128(z1, z16); + z2 = _mm512_aesdec_epi128(z2, z16); + z3 = _mm512_aesdec_epi128(z3, z16); + z0 = _mm512_aesdec_epi128(z0, z17); + z1 = _mm512_aesdec_epi128(z1, z17); + z2 = _mm512_aesdec_epi128(z2, z17); + z3 = _mm512_aesdec_epi128(z3, z17); + zf3 = (word32)r8; + zf4 = 0xb; + z7 = z18; + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_ECB_decrypt_avx512_256_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z18); + z1 = _mm512_aesdec_epi128(z1, z18); + z2 = _mm512_aesdec_epi128(z2, z18); + z3 = _mm512_aesdec_epi128(z3, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z1 = _mm512_aesdec_epi128(z1, z19); + z2 = _mm512_aesdec_epi128(z2, z19); + z3 = _mm512_aesdec_epi128(z3, z19); + zf5 = (word32)r8; + zf6 = 0xd; + z7 = z20; + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_ECB_decrypt_avx512_256_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z20); + z1 = _mm512_aesdec_epi128(z1, z20); + z2 = _mm512_aesdec_epi128(z2, z20); + z3 = _mm512_aesdec_epi128(z3, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z1 = _mm512_aesdec_epi128(z1, z21); + z2 = _mm512_aesdec_epi128(z2, z21); + z3 = _mm512_aesdec_epi128(z3, z21); + z7 = z22; +L_AES_ECB_decrypt_avx512_256_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z7); + z1 = _mm512_aesdeclast_epi128(z1, z7); + z2 = _mm512_aesdeclast_epi128(z2, z7); + z3 = _mm512_aesdeclast_epi128(z3, z7); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z1); + _mm512_storeu_si512((void*)WC_PW(r11, 128), z2); + _mm512_storeu_si512((void*)WC_PW(r11, 192), z3); + rax = (word32)((word32)rax + 0x100); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_avx512_dec_256; + } +L_AES_ECB_decrypt_avx512_done_256: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 - (word32)rax); + if (((word32)r9) < (0x80)) { + goto L_AES_ECB_decrypt_avx512_done_128; + } + /* 128 bytes of input */ + /* aes_ecb_dec_128 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + /* aes_dec_block */ + z0 = _mm512_xor_si512(z0, z8); + z1 = _mm512_xor_si512(z1, z8); + z0 = _mm512_aesdec_epi128(z0, z9); + z1 = _mm512_aesdec_epi128(z1, z9); + z0 = _mm512_aesdec_epi128(z0, z10); + z1 = _mm512_aesdec_epi128(z1, z10); + z0 = _mm512_aesdec_epi128(z0, z11); + z1 = _mm512_aesdec_epi128(z1, z11); + z0 = _mm512_aesdec_epi128(z0, z12); + z1 = _mm512_aesdec_epi128(z1, z12); + z0 = _mm512_aesdec_epi128(z0, z13); + z1 = _mm512_aesdec_epi128(z1, z13); + z0 = _mm512_aesdec_epi128(z0, z14); + z1 = _mm512_aesdec_epi128(z1, z14); + z0 = _mm512_aesdec_epi128(z0, z15); + z1 = _mm512_aesdec_epi128(z1, z15); + z0 = _mm512_aesdec_epi128(z0, z16); + z1 = _mm512_aesdec_epi128(z1, z16); + z0 = _mm512_aesdec_epi128(z0, z17); + z1 = _mm512_aesdec_epi128(z1, z17); + zf7 = (word32)r8; + zf8 = 0xb; + z7 = z18; + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_ECB_decrypt_avx512_128_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z18); + z1 = _mm512_aesdec_epi128(z1, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z1 = _mm512_aesdec_epi128(z1, z19); + zf9 = (word32)r8; + zf10 = 0xd; + z7 = z20; + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_ECB_decrypt_avx512_128_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z20); + z1 = _mm512_aesdec_epi128(z1, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z1 = _mm512_aesdec_epi128(z1, z21); + z7 = z22; +L_AES_ECB_decrypt_avx512_128_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z7); + z1 = _mm512_aesdeclast_epi128(z1, z7); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z1); + rax = (word32)((word32)rax + 0x80); +L_AES_ECB_decrypt_avx512_done_128: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 & 0xffffffc0); + if (((word32)rax) == ((word32)r9)) { + goto L_AES_ECB_decrypt_avx512_done_64; + } +L_AES_ECB_decrypt_avx512_dec_64: + /* 64 bytes of input */ + /* aes_ecb_dec_64 */ + r10 = (word64)(rdi + rax); + r11 = (word64)(rsi + rax); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + /* aes_dec_block */ + z0 = _mm512_xor_si512(z0, z8); + z0 = _mm512_aesdec_epi128(z0, z9); + z0 = _mm512_aesdec_epi128(z0, z10); + z0 = _mm512_aesdec_epi128(z0, z11); + z0 = _mm512_aesdec_epi128(z0, z12); + z0 = _mm512_aesdec_epi128(z0, z13); + z0 = _mm512_aesdec_epi128(z0, z14); + z0 = _mm512_aesdec_epi128(z0, z15); + z0 = _mm512_aesdec_epi128(z0, z16); + z0 = _mm512_aesdec_epi128(z0, z17); + zf11 = (word32)r8; + zf12 = 0xb; + z7 = z18; + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_ECB_decrypt_avx512_64_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + zf13 = (word32)r8; + zf14 = 0xd; + z7 = z20; + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_ECB_decrypt_avx512_64_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z7 = z22; +L_AES_ECB_decrypt_avx512_64_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z7); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z0); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_avx512_dec_64; + } +L_AES_ECB_decrypt_avx512_done_64: + r9 = (word32)((word32)rdx); + r9 = (word32)((word32)r9 - (word32)rax); + if (((word32)r9) < (0x20)) { + goto L_AES_ECB_decrypt_avx512_done_32; + } + /* 32 bytes of input */ + r10 = (word64)(rdi + rax); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0))); + /* aes_dec_block */ + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z8))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z9))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z10))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z11))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z12))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z13))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z14))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z15))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z16))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z17))); + zf15 = (word32)r8; + zf16 = 0xb; + z7 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z18)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_ECB_decrypt_avx512_32_aes_dec_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z18))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z19))); + zf17 = (word32)r8; + zf18 = 0xd; + z7 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z20)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_ECB_decrypt_avx512_32_aes_dec_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z20))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z21))); + z7 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z22)); +L_AES_ECB_decrypt_avx512_32_aes_dec_block_last: + z0 = _mm512_zextsi256_si512(_mm256_aesdeclast_epi128(_mm512_castsi512_si256( + z0), _mm512_castsi512_si256(z7))); + r10 = (word64)(rsi + rax); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), _mm512_castsi512_si256(z0)); + rax = (word32)((word32)rax + 0x20); +L_AES_ECB_decrypt_avx512_done_32: + zf19 = (word32)rax; + zf20 = (word32)rdx; + r9 = (word32)((word32)rdx); + if ((zf19) == (zf20)) { + goto L_AES_ECB_decrypt_avx512_done_dec; + } + r9 = (word32)((word32)r9 & 0xfffffff0); +L_AES_ECB_decrypt_avx512_dec_16: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + /* aes_dec_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 128)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 144)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + zf21 = (word32)r8; + zf22 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 160)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_ECB_decrypt_avx512_16_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 176)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x6)); + zf23 = (word32)r8; + zf24 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 192)); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_ECB_decrypt_avx512_16_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 208)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x6)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 224)); +L_AES_ECB_decrypt_avx512_16_aes_dec_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesdeclast_si128(_mm512_castsi512_si128(z0), + x5)); + r10 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), _mm512_castsi512_si128(z0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r9)) { + goto L_AES_ECB_decrypt_avx512_dec_16; + } +L_AES_ECB_decrypt_avx512_done_dec: + ; +} + +WC_X64I_TARGET("aes,avx,avx512f,avx512vl") +WOLFSSL_LOCAL void AES_CBC_encrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10 = 0, r11 = 0; + __m128i x0, x1 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ivec; + rcx = (word64)(word64)length; + r8 = (word64)(size_t)KS; + r9 = (word64)(word32)nr; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + rax = (word32)(0); + if (((word32)rax) == ((word32)rcx)) { + goto L_AES_CBC_encrypt_avx512_done; + } +L_AES_CBC_encrypt_avx512_loop: + /* 16 bytes of input */ + r10 = (word64)(rdi + rax); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_ternarylogic_epi64(x1, x0, _mm_loadu_si128((const __m128i*)WC_PR( + r8, 0)), 0x96); + /* aes_enc_block */ + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x1 = _mm_aesenc_si128(x1, x3); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x1 = _mm_aesenc_si128(x1, x3); + zf1 = (word32)r9; + zf2 = 0xb; + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_CBC_encrypt_avx512_aes_enc_block_last; + } + x1 = _mm_aesenc_si128(x1, x3); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x1 = _mm_aesenc_si128(x1, x4); + zf3 = (word32)r9; + zf4 = 0xd; + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CBC_encrypt_avx512_aes_enc_block_last; + } + x1 = _mm_aesenc_si128(x1, x3); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x1 = _mm_aesenc_si128(x1, x4); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_encrypt_avx512_aes_enc_block_last: + x1 = _mm_aesenclast_si128(x1, x3); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), x1); + x0 = x1; + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)rcx)) { + goto L_AES_CBC_encrypt_avx512_loop; + } +L_AES_CBC_encrypt_avx512_done: + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); +} + +WC_X64I_TARGET("aes,vaes,avx512f,avx512vl") +WOLFSSL_LOCAL void AES_CBC_decrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned char* ivec, unsigned long length, + const unsigned char* KS, int nr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10 = 0, r11 = 0, r12 = 0; + __m128i x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(); + __m256i y8; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z9 = _mm512_setzero_si512(), z10 = _mm512_setzero_si512(), + z11 = _mm512_setzero_si512(), z12 = _mm512_setzero_si512(), + z13 = _mm512_setzero_si512(), z14 = _mm512_setzero_si512(), + z15 = _mm512_setzero_si512(), z16 = _mm512_setzero_si512(), + z17 = _mm512_setzero_si512(), z18 = _mm512_setzero_si512(), + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(), + z23 = _mm512_setzero_si512(), z24 = _mm512_setzero_si512(), + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(), + z27 = _mm512_setzero_si512(), z28 = _mm512_setzero_si512(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ivec; + rcx = (word64)(word64)length; + r8 = (word64)(size_t)KS; + r9 = (word64)(word32)nr; + + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + rax = (word32)(0); + if (((word32)rcx) < (0x20)) { + goto L_AES_CBC_decrypt_avx512_done_32; + } + z14 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + z16 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + z17 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + z18 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + z19 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + z21 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + z23 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + z24 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)((word32)r9) < (sword32)(0xb)) { + goto L_AES_CBC_decrypt_avx512_key_cached; + } + z25 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + z26 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)((word32)r9) < (sword32)(0xd)) { + goto L_AES_CBC_decrypt_avx512_key_cached; + } + z27 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + z28 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_CBC_decrypt_avx512_key_cached: + zf1 = (word32)rcx; + zf2 = 0x100; + r10 = (word32)((word32)rcx); + if ((zf1) < (zf2)) { + goto L_AES_CBC_decrypt_avx512_done_256; + } + r10 = (word32)((word32)r10 & 0xffffff00); +L_AES_CBC_decrypt_avx512_dec_256: + /* 256 bytes of input */ + /* aes_cbc_dec_256 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + z0 = _mm512_loadu_si512((const void*)WC_PR(r11, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r11, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r11, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r11, 192)); + z10 = _mm512_shuffle_i64x2(z0, z0, 0x90); + z10 = _mm512_inserti32x4(z10, _mm256_castsi256_si128(y8), 0); + z11 = _mm512_loadu_si512((const void*)WC_PR(r11, 48)); + z12 = _mm512_loadu_si512((const void*)WC_PR(r11, 112)); + z13 = _mm512_loadu_si512((const void*)WC_PR(r11, 176)); + y8 = _mm256_zextsi128_si256(_mm512_extracti32x4_epi32(z3, 3)); + /* aes_dec_block */ + z0 = _mm512_xor_si512(z0, z14); + z1 = _mm512_xor_si512(z1, z14); + z2 = _mm512_xor_si512(z2, z14); + z3 = _mm512_xor_si512(z3, z14); + z0 = _mm512_aesdec_epi128(z0, z15); + z1 = _mm512_aesdec_epi128(z1, z15); + z2 = _mm512_aesdec_epi128(z2, z15); + z3 = _mm512_aesdec_epi128(z3, z15); + z0 = _mm512_aesdec_epi128(z0, z16); + z1 = _mm512_aesdec_epi128(z1, z16); + z2 = _mm512_aesdec_epi128(z2, z16); + z3 = _mm512_aesdec_epi128(z3, z16); + z0 = _mm512_aesdec_epi128(z0, z17); + z1 = _mm512_aesdec_epi128(z1, z17); + z2 = _mm512_aesdec_epi128(z2, z17); + z3 = _mm512_aesdec_epi128(z3, z17); + z0 = _mm512_aesdec_epi128(z0, z18); + z1 = _mm512_aesdec_epi128(z1, z18); + z2 = _mm512_aesdec_epi128(z2, z18); + z3 = _mm512_aesdec_epi128(z3, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z1 = _mm512_aesdec_epi128(z1, z19); + z2 = _mm512_aesdec_epi128(z2, z19); + z3 = _mm512_aesdec_epi128(z3, z19); + z0 = _mm512_aesdec_epi128(z0, z20); + z1 = _mm512_aesdec_epi128(z1, z20); + z2 = _mm512_aesdec_epi128(z2, z20); + z3 = _mm512_aesdec_epi128(z3, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z1 = _mm512_aesdec_epi128(z1, z21); + z2 = _mm512_aesdec_epi128(z2, z21); + z3 = _mm512_aesdec_epi128(z3, z21); + z0 = _mm512_aesdec_epi128(z0, z22); + z1 = _mm512_aesdec_epi128(z1, z22); + z2 = _mm512_aesdec_epi128(z2, z22); + z3 = _mm512_aesdec_epi128(z3, z22); + z0 = _mm512_aesdec_epi128(z0, z23); + z1 = _mm512_aesdec_epi128(z1, z23); + z2 = _mm512_aesdec_epi128(z2, z23); + z3 = _mm512_aesdec_epi128(z3, z23); + zf3 = (word32)r9; + zf4 = 0xb; + z9 = z24; + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CBC_decrypt_avx512_256_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z24); + z1 = _mm512_aesdec_epi128(z1, z24); + z2 = _mm512_aesdec_epi128(z2, z24); + z3 = _mm512_aesdec_epi128(z3, z24); + z0 = _mm512_aesdec_epi128(z0, z25); + z1 = _mm512_aesdec_epi128(z1, z25); + z2 = _mm512_aesdec_epi128(z2, z25); + z3 = _mm512_aesdec_epi128(z3, z25); + zf5 = (word32)r9; + zf6 = 0xd; + z9 = z26; + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_CBC_decrypt_avx512_256_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z26); + z1 = _mm512_aesdec_epi128(z1, z26); + z2 = _mm512_aesdec_epi128(z2, z26); + z3 = _mm512_aesdec_epi128(z3, z26); + z0 = _mm512_aesdec_epi128(z0, z27); + z1 = _mm512_aesdec_epi128(z1, z27); + z2 = _mm512_aesdec_epi128(z2, z27); + z3 = _mm512_aesdec_epi128(z3, z27); + z9 = z28; +L_AES_CBC_decrypt_avx512_256_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z9); + z1 = _mm512_aesdeclast_epi128(z1, z9); + z2 = _mm512_aesdeclast_epi128(z2, z9); + z3 = _mm512_aesdeclast_epi128(z3, z9); + z0 = _mm512_xor_si512(z0, z10); + z1 = _mm512_xor_si512(z1, z11); + z2 = _mm512_xor_si512(z2, z12); + z3 = _mm512_xor_si512(z3, z13); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z1); + _mm512_storeu_si512((void*)WC_PW(r12, 128), z2); + _mm512_storeu_si512((void*)WC_PW(r12, 192), z3); + rax = (word32)((word32)rax + 0x100); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_avx512_dec_256; + } +L_AES_CBC_decrypt_avx512_done_256: + r10 = (word32)((word32)rcx); + r10 = (word32)((word32)r10 - (word32)rax); + if (((word32)r10) < (0x80)) { + goto L_AES_CBC_decrypt_avx512_done_128; + } + /* 128 bytes of input */ + /* aes_cbc_dec_128 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + z0 = _mm512_loadu_si512((const void*)WC_PR(r11, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r11, 64)); + z10 = _mm512_shuffle_i64x2(z0, z0, 0x90); + z10 = _mm512_inserti32x4(z10, _mm256_castsi256_si128(y8), 0); + z11 = _mm512_loadu_si512((const void*)WC_PR(r11, 48)); + y8 = _mm256_zextsi128_si256(_mm512_extracti32x4_epi32(z1, 3)); + /* aes_dec_block */ + z0 = _mm512_xor_si512(z0, z14); + z1 = _mm512_xor_si512(z1, z14); + z0 = _mm512_aesdec_epi128(z0, z15); + z1 = _mm512_aesdec_epi128(z1, z15); + z0 = _mm512_aesdec_epi128(z0, z16); + z1 = _mm512_aesdec_epi128(z1, z16); + z0 = _mm512_aesdec_epi128(z0, z17); + z1 = _mm512_aesdec_epi128(z1, z17); + z0 = _mm512_aesdec_epi128(z0, z18); + z1 = _mm512_aesdec_epi128(z1, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z1 = _mm512_aesdec_epi128(z1, z19); + z0 = _mm512_aesdec_epi128(z0, z20); + z1 = _mm512_aesdec_epi128(z1, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z1 = _mm512_aesdec_epi128(z1, z21); + z0 = _mm512_aesdec_epi128(z0, z22); + z1 = _mm512_aesdec_epi128(z1, z22); + z0 = _mm512_aesdec_epi128(z0, z23); + z1 = _mm512_aesdec_epi128(z1, z23); + zf7 = (word32)r9; + zf8 = 0xb; + z9 = z24; + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_CBC_decrypt_avx512_128_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z24); + z1 = _mm512_aesdec_epi128(z1, z24); + z0 = _mm512_aesdec_epi128(z0, z25); + z1 = _mm512_aesdec_epi128(z1, z25); + zf9 = (word32)r9; + zf10 = 0xd; + z9 = z26; + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_CBC_decrypt_avx512_128_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z26); + z1 = _mm512_aesdec_epi128(z1, z26); + z0 = _mm512_aesdec_epi128(z0, z27); + z1 = _mm512_aesdec_epi128(z1, z27); + z9 = z28; +L_AES_CBC_decrypt_avx512_128_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z9); + z1 = _mm512_aesdeclast_epi128(z1, z9); + z0 = _mm512_xor_si512(z0, z10); + z1 = _mm512_xor_si512(z1, z11); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z1); + rax = (word32)((word32)rax + 0x80); +L_AES_CBC_decrypt_avx512_done_128: + r10 = (word32)((word32)rcx); + r10 = (word32)((word32)r10 & 0xffffffc0); + if (((word32)rax) == ((word32)r10)) { + goto L_AES_CBC_decrypt_avx512_done_64; + } +L_AES_CBC_decrypt_avx512_dec_64: + /* 64 bytes of input */ + /* aes_cbc_dec_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + z0 = _mm512_loadu_si512((const void*)WC_PR(r11, 0)); + z10 = _mm512_shuffle_i64x2(z0, z0, 0x90); + z10 = _mm512_inserti32x4(z10, _mm256_castsi256_si128(y8), 0); + y8 = _mm256_zextsi128_si256(_mm512_extracti32x4_epi32(z0, 3)); + /* aes_dec_block */ + z0 = _mm512_xor_si512(z0, z14); + z0 = _mm512_aesdec_epi128(z0, z15); + z0 = _mm512_aesdec_epi128(z0, z16); + z0 = _mm512_aesdec_epi128(z0, z17); + z0 = _mm512_aesdec_epi128(z0, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z0 = _mm512_aesdec_epi128(z0, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z0 = _mm512_aesdec_epi128(z0, z22); + z0 = _mm512_aesdec_epi128(z0, z23); + zf11 = (word32)r9; + zf12 = 0xb; + z9 = z24; + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_CBC_decrypt_avx512_64_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z24); + z0 = _mm512_aesdec_epi128(z0, z25); + zf13 = (word32)r9; + zf14 = 0xd; + z9 = z26; + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_CBC_decrypt_avx512_64_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z26); + z0 = _mm512_aesdec_epi128(z0, z27); + z9 = z28; +L_AES_CBC_decrypt_avx512_64_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z9); + z0 = _mm512_xor_si512(z0, z10); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z0); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_avx512_dec_64; + } +L_AES_CBC_decrypt_avx512_done_64: + r10 = (word32)((word32)rcx); + r10 = (word32)((word32)r10 - (word32)rax); + if (((word32)r10) < (0x20)) { + goto L_AES_CBC_decrypt_avx512_done_32; + } + /* 32 bytes of input */ + r11 = (word64)(rdi + rax); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + z10 = _mm512_zextsi256_si512(_mm256_inserti128_si256(y8, + _mm512_castsi512_si128(z0), 1)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(_mm512_castsi512_si256( + z0), 1)); + /* aes_dec_block */ + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z14))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z15))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z16))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z17))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z18))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z19))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z20))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z21))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z22))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z23))); + zf15 = (word32)r9; + zf16 = 0xb; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z24)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_CBC_decrypt_avx512_32_aes_dec_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z24))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z25))); + zf17 = (word32)r9; + zf18 = 0xd; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z26)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_CBC_decrypt_avx512_32_aes_dec_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z26))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z27))); + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z28)); +L_AES_CBC_decrypt_avx512_32_aes_dec_block_last: + z0 = _mm512_zextsi256_si512(_mm256_aesdeclast_epi128(_mm512_castsi512_si256( + z0), _mm512_castsi512_si256(z9))); + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z10))); + r11 = (word64)(rsi + rax); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), _mm512_castsi512_si256(z0)); + rax = (word32)((word32)rax + 0x20); +L_AES_CBC_decrypt_avx512_done_32: + zf19 = (word32)rax; + zf20 = (word32)rcx; + r10 = (word32)((word32)rcx); + if ((zf19) == (zf20)) { + goto L_AES_CBC_decrypt_avx512_done_dec; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_CBC_decrypt_avx512_dec_16: + /* 16 bytes of input */ + r11 = (word64)(rdi + rax); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x7 = _mm512_castsi512_si128(z0); + /* aes_dec_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + zf21 = (word32)r9; + zf22 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_CBC_decrypt_avx512_16_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x6)); + zf23 = (word32)r9; + zf24 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_CBC_decrypt_avx512_16_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x5)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + x6)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_CBC_decrypt_avx512_16_aes_dec_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesdeclast_si128(_mm512_castsi512_si128(z0), + x5)); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(x7); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), _mm512_castsi512_si128(z0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CBC_decrypt_avx512_dec_16; + } +L_AES_CBC_decrypt_avx512_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y8)); +} + +XALIGNED(16) static const word64 L_aes_ctr_bswap_avx512[] WC_X64I_UNUSED = { + 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, +}; + +XALIGNED(32) static const word64 L_aes_ctr_inc_avx512[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0x0000000000000002ULL, 0x0000000000000000ULL, + 0x0000000000000003ULL, 0x0000000000000000ULL, + 0x0000000000000004ULL, 0x0000000000000000ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000000000008ULL, 0x0000000000000000ULL, + 0x0000000000000009ULL, 0x0000000000000000ULL, + 0x000000000000000aULL, 0x0000000000000000ULL, + 0x000000000000000bULL, 0x0000000000000000ULL, + 0x000000000000000cULL, 0x0000000000000000ULL, + 0x000000000000000dULL, 0x0000000000000000ULL, + 0x000000000000000eULL, 0x0000000000000000ULL, + 0x000000000000000fULL, 0x0000000000000000ULL, + 0x0000000000000010ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("aes,vaes,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_CTR_encrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned long length, const unsigned char* KS, int nr, + unsigned char* ctr) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10 = 0, r11 = 0, r12 = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7, z8, z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), z12, + z13, z14 = _mm512_setzero_si512(), z15 = _mm512_setzero_si512(), + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(), + z22 = _mm512_setzero_si512(), z23 = _mm512_setzero_si512(), + z24 = _mm512_setzero_si512(), z25 = _mm512_setzero_si512(), + z26 = _mm512_setzero_si512(), z27 = _mm512_setzero_si512(), + z28 = _mm512_setzero_si512(), z29 = _mm512_setzero_si512(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(word64)length; + rcx = (word64)(size_t)KS; + r8 = (word64)(word32)nr; + r9 = (word64)(size_t)ctr; + + z8 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_bswap_avx512, 0))); + z7 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + z7 = _mm512_shuffle_epi8(z7, z8); + z12 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_inc_avx512, 64))); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_inc_avx512, 16))); + rax = (word32)(0); + if (((word32)rdx) < (0x20)) { + goto L_AES_CTR_encrypt_avx512_done_32; + } + z15 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 0))); + z16 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + z17 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + z18 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + z19 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + z21 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + z23 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + z24 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + z25 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)((word32)r8) < (sword32)(0xb)) { + goto L_AES_CTR_encrypt_avx512_key_cached; + } + z26 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + z27 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)((word32)r8) < (sword32)(0xd)) { + goto L_AES_CTR_encrypt_avx512_key_cached; + } + z28 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + z29 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_CTR_encrypt_avx512_key_cached: + zf1 = (word32)rdx; + zf2 = 0x100; + r10 = (word32)((word32)rdx); + if ((zf1) < (zf2)) { + goto L_AES_CTR_encrypt_avx512_done_256; + } + r10 = (word32)((word32)r10 & 0xffffff00); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_inc_avx512, 256))); + z9 = z7; + z4 = _mm512_add_epi64(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_ctr_inc_avx512, 0))); + z9 = _mm512_ternarylogic_epi64(z9, z4, _mm512_loadu_si512(( + const void*)WC_PR(L_aes_ctr_inc_avx512, 0)), 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z4 = _mm512_add_epi64(z4, z9); + z9 = z7; + z5 = _mm512_add_epi64(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_ctr_inc_avx512, 64))); + z9 = _mm512_ternarylogic_epi64(z9, z5, _mm512_loadu_si512(( + const void*)WC_PR(L_aes_ctr_inc_avx512, 64)), 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_add_epi64(z5, z9); + z9 = z7; + z6 = _mm512_add_epi64(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_ctr_inc_avx512, 128))); + z9 = _mm512_ternarylogic_epi64(z9, z6, _mm512_loadu_si512(( + const void*)WC_PR(L_aes_ctr_inc_avx512, 128)), 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_add_epi64(z6, z9); + z9 = z7; + z7 = _mm512_add_epi64(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_ctr_inc_avx512, 192))); + z9 = _mm512_ternarylogic_epi64(z9, z7, _mm512_loadu_si512(( + const void*)WC_PR(L_aes_ctr_inc_avx512, 192)), 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_add_epi64(z7, z9); +L_AES_CTR_encrypt_avx512_enc_256: + /* 256 bytes of input */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + z0 = _mm512_shuffle_epi8(z4, z8); + z1 = _mm512_shuffle_epi8(z5, z8); + z2 = _mm512_shuffle_epi8(z6, z8); + z3 = _mm512_shuffle_epi8(z7, z8); + z9 = z4; + z4 = _mm512_add_epi64(z4, z10); + z9 = _mm512_ternarylogic_epi64(z9, z4, z10, 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z4 = _mm512_add_epi64(z4, z9); + z9 = z5; + z5 = _mm512_add_epi64(z5, z10); + z9 = _mm512_ternarylogic_epi64(z9, z5, z10, 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_add_epi64(z5, z9); + z9 = z6; + z6 = _mm512_add_epi64(z6, z10); + z9 = _mm512_ternarylogic_epi64(z9, z6, z10, 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_add_epi64(z6, z9); + z9 = z7; + z7 = _mm512_add_epi64(z7, z10); + z9 = _mm512_ternarylogic_epi64(z9, z7, z10, 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_add_epi64(z7, z9); + /* aes_enc_block */ + z0 = _mm512_xor_si512(z0, z15); + z1 = _mm512_xor_si512(z1, z15); + z2 = _mm512_xor_si512(z2, z15); + z3 = _mm512_xor_si512(z3, z15); + z0 = _mm512_aesenc_epi128(z0, z16); + z1 = _mm512_aesenc_epi128(z1, z16); + z2 = _mm512_aesenc_epi128(z2, z16); + z3 = _mm512_aesenc_epi128(z3, z16); + z0 = _mm512_aesenc_epi128(z0, z17); + z1 = _mm512_aesenc_epi128(z1, z17); + z2 = _mm512_aesenc_epi128(z2, z17); + z3 = _mm512_aesenc_epi128(z3, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z1 = _mm512_aesenc_epi128(z1, z18); + z2 = _mm512_aesenc_epi128(z2, z18); + z3 = _mm512_aesenc_epi128(z3, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z1 = _mm512_aesenc_epi128(z1, z19); + z2 = _mm512_aesenc_epi128(z2, z19); + z3 = _mm512_aesenc_epi128(z3, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z1 = _mm512_aesenc_epi128(z1, z20); + z2 = _mm512_aesenc_epi128(z2, z20); + z3 = _mm512_aesenc_epi128(z3, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z1 = _mm512_aesenc_epi128(z1, z21); + z2 = _mm512_aesenc_epi128(z2, z21); + z3 = _mm512_aesenc_epi128(z3, z21); + z0 = _mm512_aesenc_epi128(z0, z22); + z1 = _mm512_aesenc_epi128(z1, z22); + z2 = _mm512_aesenc_epi128(z2, z22); + z3 = _mm512_aesenc_epi128(z3, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z1 = _mm512_aesenc_epi128(z1, z23); + z2 = _mm512_aesenc_epi128(z2, z23); + z3 = _mm512_aesenc_epi128(z3, z23); + z0 = _mm512_aesenc_epi128(z0, z24); + z1 = _mm512_aesenc_epi128(z1, z24); + z2 = _mm512_aesenc_epi128(z2, z24); + z3 = _mm512_aesenc_epi128(z3, z24); + zf3 = (word32)r8; + zf4 = 0xb; + z14 = z25; + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_CTR_encrypt_avx512_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z25); + z1 = _mm512_aesenc_epi128(z1, z25); + z2 = _mm512_aesenc_epi128(z2, z25); + z3 = _mm512_aesenc_epi128(z3, z25); + z0 = _mm512_aesenc_epi128(z0, z26); + z1 = _mm512_aesenc_epi128(z1, z26); + z2 = _mm512_aesenc_epi128(z2, z26); + z3 = _mm512_aesenc_epi128(z3, z26); + zf5 = (word32)r8; + zf6 = 0xd; + z14 = z27; + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_CTR_encrypt_avx512_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z27); + z1 = _mm512_aesenc_epi128(z1, z27); + z2 = _mm512_aesenc_epi128(z2, z27); + z3 = _mm512_aesenc_epi128(z3, z27); + z0 = _mm512_aesenc_epi128(z0, z28); + z1 = _mm512_aesenc_epi128(z1, z28); + z2 = _mm512_aesenc_epi128(z2, z28); + z3 = _mm512_aesenc_epi128(z3, z28); + z14 = z29; +L_AES_CTR_encrypt_avx512_256_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z14); + z1 = _mm512_aesenclast_epi128(z1, z14); + z2 = _mm512_aesenclast_epi128(z2, z14); + z3 = _mm512_aesenclast_epi128(z3, z14); + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(r11, 0))); + z1 = _mm512_xor_si512(z1, _mm512_loadu_si512((const void*)WC_PR(r11, 64))); + z2 = _mm512_xor_si512(z2, _mm512_loadu_si512((const void*)WC_PR(r11, 128))); + z3 = _mm512_xor_si512(z3, _mm512_loadu_si512((const void*)WC_PR(r11, 192))); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z1); + _mm512_storeu_si512((void*)WC_PW(r12, 128), z2); + _mm512_storeu_si512((void*)WC_PW(r12, 192), z3); + rax = (word32)((word32)rax + 0x100); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_avx512_enc_256; + } + z7 = _mm512_shuffle_i64x2(z4, z4, 0); +L_AES_CTR_encrypt_avx512_done_256: + r10 = (word32)((word32)rdx); + r10 = (word32)((word32)r10 - (word32)rax); + if (((word32)r10) < (0x80)) { + goto L_AES_CTR_encrypt_avx512_done_128; + } + /* 128 bytes of input */ + z11 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_inc_avx512, 128))); + /* aes_ctr_enc_128 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + z0 = _mm512_add_epi64(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_ctr_inc_avx512, 0))); + z9 = z7; + z9 = _mm512_ternarylogic_epi64(z9, z0, _mm512_loadu_si512(( + const void*)WC_PR(L_aes_ctr_inc_avx512, 0)), 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z0 = _mm512_add_epi64(z0, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z1 = _mm512_add_epi64(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_ctr_inc_avx512, 64))); + z9 = z7; + z9 = _mm512_ternarylogic_epi64(z9, z1, _mm512_loadu_si512(( + const void*)WC_PR(L_aes_ctr_inc_avx512, 64)), 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z1 = _mm512_add_epi64(z1, z9); + z1 = _mm512_shuffle_epi8(z1, z8); + z9 = z7; + z7 = _mm512_add_epi64(z7, z11); + z9 = _mm512_ternarylogic_epi64(z9, z7, z11, 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_add_epi64(z7, z9); + /* aes_enc_block */ + z0 = _mm512_xor_si512(z0, z15); + z1 = _mm512_xor_si512(z1, z15); + z0 = _mm512_aesenc_epi128(z0, z16); + z1 = _mm512_aesenc_epi128(z1, z16); + z0 = _mm512_aesenc_epi128(z0, z17); + z1 = _mm512_aesenc_epi128(z1, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z1 = _mm512_aesenc_epi128(z1, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z1 = _mm512_aesenc_epi128(z1, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z1 = _mm512_aesenc_epi128(z1, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z1 = _mm512_aesenc_epi128(z1, z21); + z0 = _mm512_aesenc_epi128(z0, z22); + z1 = _mm512_aesenc_epi128(z1, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z1 = _mm512_aesenc_epi128(z1, z23); + z0 = _mm512_aesenc_epi128(z0, z24); + z1 = _mm512_aesenc_epi128(z1, z24); + zf7 = (word32)r8; + zf8 = 0xb; + z14 = z25; + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_CTR_encrypt_avx512_128_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z25); + z1 = _mm512_aesenc_epi128(z1, z25); + z0 = _mm512_aesenc_epi128(z0, z26); + z1 = _mm512_aesenc_epi128(z1, z26); + zf9 = (word32)r8; + zf10 = 0xd; + z14 = z27; + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_CTR_encrypt_avx512_128_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z27); + z1 = _mm512_aesenc_epi128(z1, z27); + z0 = _mm512_aesenc_epi128(z0, z28); + z1 = _mm512_aesenc_epi128(z1, z28); + z14 = z29; +L_AES_CTR_encrypt_avx512_128_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z14); + z1 = _mm512_aesenclast_epi128(z1, z14); + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(r11, 0))); + z1 = _mm512_xor_si512(z1, _mm512_loadu_si512((const void*)WC_PR(r11, 64))); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z1); + rax = (word32)((word32)rax + 0x80); +L_AES_CTR_encrypt_avx512_done_128: + r10 = (word32)((word32)rdx); + r10 = (word32)((word32)r10 & 0xffffffc0); + if (((word32)rax) == ((word32)r10)) { + goto L_AES_CTR_encrypt_avx512_done_64; + } +L_AES_CTR_encrypt_avx512_enc_64: + /* 64 bytes of input */ + /* aes_ctr_enc_64 */ + r11 = (word64)(rdi + rax); + r12 = (word64)(rsi + rax); + z0 = _mm512_add_epi64(z7, _mm512_loadu_si512((const void*)WC_PR( + L_aes_ctr_inc_avx512, 0))); + z9 = z7; + z9 = _mm512_ternarylogic_epi64(z9, z0, _mm512_loadu_si512(( + const void*)WC_PR(L_aes_ctr_inc_avx512, 0)), 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z0 = _mm512_add_epi64(z0, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z9 = z7; + z7 = _mm512_add_epi64(z7, z12); + z9 = _mm512_ternarylogic_epi64(z9, z7, z12, 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_add_epi64(z7, z9); + /* aes_enc_block */ + z0 = _mm512_xor_si512(z0, z15); + z0 = _mm512_aesenc_epi128(z0, z16); + z0 = _mm512_aesenc_epi128(z0, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z0 = _mm512_aesenc_epi128(z0, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z0 = _mm512_aesenc_epi128(z0, z24); + zf11 = (word32)r8; + zf12 = 0xb; + z14 = z25; + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_CTR_encrypt_avx512_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z25); + z0 = _mm512_aesenc_epi128(z0, z26); + zf13 = (word32)r8; + zf14 = 0xd; + z14 = z27; + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_CTR_encrypt_avx512_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z27); + z0 = _mm512_aesenc_epi128(z0, z28); + z14 = z29; +L_AES_CTR_encrypt_avx512_64_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z14); + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(r11, 0))); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z0); + rax = (word32)((word32)rax + 0x40); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_avx512_enc_64; + } +L_AES_CTR_encrypt_avx512_done_64: + r10 = (word32)((word32)rdx); + r10 = (word32)((word32)r10 - (word32)rax); + if (((word32)r10) < (0x20)) { + goto L_AES_CTR_encrypt_avx512_done_32; + } + /* 32 bytes of input */ + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_aes_ctr_inc_avx512, 32))); + z0 = _mm512_zextsi256_si512(_mm256_add_epi64(_mm512_castsi512_si256(z7), + _mm256_loadu_si256((const __m256i*)WC_PR(L_aes_ctr_inc_avx512, 0)))); + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z7)); + z9 = _mm512_zextsi256_si512(_mm256_ternarylogic_epi64( + _mm512_castsi512_si256(z9), _mm512_castsi512_si256(z0), + _mm256_loadu_si256((const __m256i*)WC_PR(L_aes_ctr_inc_avx512, 0)), + 0xb2)); + z9 = _mm512_zextsi256_si512(_mm256_srli_epi64(_mm512_castsi512_si256(z9), + 63)); + z9 = _mm512_zextsi256_si512(_mm256_bslli_epi128(_mm512_castsi512_si256(z9), + 8)); + z0 = _mm512_zextsi256_si512(_mm256_add_epi64(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z9))); + z0 = _mm512_zextsi256_si512(_mm256_shuffle_epi8(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z8))); + z9 = z7; + z7 = _mm512_add_epi64(z7, z4); + z9 = _mm512_ternarylogic_epi64(z9, z7, z4, 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_add_epi64(z7, z9); + /* aes_enc_block */ + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z15))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z16))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z17))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z18))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z19))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z20))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z21))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z22))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z23))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z24))); + zf15 = (word32)r8; + zf16 = 0xb; + z14 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z25)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_CTR_encrypt_avx512_32_aes_enc_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z25))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z26))); + zf17 = (word32)r8; + zf18 = 0xd; + z14 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z27)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_CTR_encrypt_avx512_32_aes_enc_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z27))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z28))); + z14 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z29)); +L_AES_CTR_encrypt_avx512_32_aes_enc_block_last: + z0 = _mm512_zextsi256_si512(_mm256_aesenclast_epi128(_mm512_castsi512_si256( + z0), _mm512_castsi512_si256(z14))); + r11 = (word64)(rdi + rax); + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm256_loadu_si256((const __m256i*)WC_PR(r11, 0)))); + r11 = (word64)(rsi + rax); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), _mm512_castsi512_si256(z0)); + rax = (word32)((word32)rax + 0x20); +L_AES_CTR_encrypt_avx512_done_32: + zf19 = (word32)rax; + zf20 = (word32)rdx; + r10 = (word32)((word32)rdx); + if ((zf19) == (zf20)) { + goto L_AES_CTR_encrypt_avx512_done_enc; + } + r10 = (word32)((word32)r10 & 0xfffffff0); +L_AES_CTR_encrypt_avx512_enc_16: + /* 16 bytes of input */ + z0 = _mm512_zextsi128_si512(_mm_shuffle_epi8(_mm512_castsi512_si128(z7), + _mm512_castsi512_si128(z8))); + z9 = z7; + z7 = _mm512_add_epi64(z7, z13); + z9 = _mm512_ternarylogic_epi64(z9, z7, z13, 0xb2); + z9 = _mm512_srli_epi64(z9, 63); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_add_epi64(z7, z9); + /* aes_enc_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf21 = (word32)r8; + zf22 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160))); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_CTR_encrypt_avx512_16_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf23 = (word32)r8; + zf24 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_CTR_encrypt_avx512_16_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224))); +L_AES_CTR_encrypt_avx512_16_aes_enc_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + r11 = (word64)(rdi + rax); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r11, 0)))); + r11 = (word64)(rsi + rax); + _mm_storeu_si128((__m128i*)WC_PW(r11, 0), _mm512_castsi512_si128(z0)); + rax = (word32)((word32)rax + 0x10); + if (((word32)rax) < ((word32)r10)) { + goto L_AES_CTR_encrypt_avx512_enc_16; + } +L_AES_CTR_encrypt_avx512_done_enc: + z0 = _mm512_zextsi128_si512(_mm_shuffle_epi8(_mm512_castsi512_si128(z7), + _mm512_castsi512_si128(z8))); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); +} + +#endif /* HAVE_INTEL_AVX512 */ +#endif /* WOLFSSL_X86_64_BUILD */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/aes_xts_intrin.c b/wolfcrypt/src/aes_xts_intrin.c new file mode 100644 index 00000000000..7fda60ddf7d --- /dev/null +++ b/wolfcrypt/src/aes_xts_intrin.c @@ -0,0 +1,9735 @@ +/* aes_xts_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_AES_XTS_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_AES_XTS +#ifdef WOLFSSL_X86_64_BUILD +extern WOLFSSL_LOCAL void AES_XTS_init_aesni(const unsigned char* i, + const unsigned char* key, int nr); +extern WOLFSSL_LOCAL void AES_XTS_encrypt_aesni(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr); +extern WOLFSSL_LOCAL void AES_XTS_encrypt_update_aesni(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr); +extern WOLFSSL_LOCAL void AES_XTS_decrypt_aesni(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr); +extern WOLFSSL_LOCAL void AES_XTS_decrypt_update_aesni(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr); +#ifdef HAVE_INTEL_AVX1 +extern WOLFSSL_LOCAL void AES_XTS_init_avx1(const unsigned char* i, + const unsigned char* key, int nr); +extern WOLFSSL_LOCAL void AES_XTS_encrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr); +extern WOLFSSL_LOCAL void AES_XTS_encrypt_update_avx1(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr); +extern WOLFSSL_LOCAL void AES_XTS_decrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr); +extern WOLFSSL_LOCAL void AES_XTS_decrypt_update_avx1(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr); +#endif +#ifdef HAVE_INTEL_VAES +extern WOLFSSL_LOCAL void AES_XTS_init_vaes(const unsigned char* i, + const unsigned char* key, int nr); +extern WOLFSSL_LOCAL void AES_XTS_encrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr); +extern WOLFSSL_LOCAL void AES_XTS_encrypt_update_vaes(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr); +extern WOLFSSL_LOCAL void AES_XTS_decrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr); +extern WOLFSSL_LOCAL void AES_XTS_decrypt_update_vaes(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr); +#endif +#ifdef HAVE_INTEL_AVX512 +extern WOLFSSL_LOCAL void AES_XTS_init_avx512(const unsigned char* i, + const unsigned char* key, int nr); +extern WOLFSSL_LOCAL void AES_XTS_encrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr); +extern WOLFSSL_LOCAL void AES_XTS_encrypt_update_avx512(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr); +extern WOLFSSL_LOCAL void AES_XTS_decrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr); +extern WOLFSSL_LOCAL void AES_XTS_decrypt_update_avx512(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr); + +#endif +#endif +#endif +#ifdef WOLFSSL_AES_XTS +#ifdef WOLFSSL_X86_64_BUILD +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_XTS_init_aesni(const unsigned char* i, + const unsigned char* key, int nr) +{ + word64 rdi, rsi, rdx; + __m128i x0, x2, x3 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)i; + rsi = (word64)(size_t)key; + rdx = (word64)(word32)nr; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x0 = _mm_aesenc_si128(x0, x2); + zf1 = (word32)rdx; + zf2 = 0xb; + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_init_aesni_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x2); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_aesenc_si128(x0, x3); + zf3 = (word32)rdx; + zf4 = 0xd; + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_init_aesni_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x2); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x0 = _mm_aesenc_si128(x0, x3); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_XTS_init_aesni_tweak_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); +} + +XALIGNED(16) static const word32 L_aes_xts_gc_xts[] WC_X64I_UNUSED = { + 0x00000087, 0x00000001, 0x00000001, 0x00000001, +}; + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_XTS_encrypt_aesni(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr) +{ + word64 rdi, rsi, rax, r12, r8, r9, r10, rsp, r13 = 0, r11 = 0, rcx = 0, + rdx = 0; + __m128i x0, x1 = _mm_setzero_si128(), x2 = _mm_setzero_si128(), + x3 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), x5, + x6 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12; + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r12 = (word64)(size_t)i; + r8 = (word64)(size_t)key; + r9 = (word64)(size_t)key2; + r10 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_xts_gc_xts, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf1 = (word32)r10; + zf2 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_encrypt_aesni_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf3 = (word32)r10; + zf4 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_encrypt_aesni_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 224)); +L_AES_XTS_encrypt_aesni_tweak_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r13 = (word32)(0); + zf5 = (word32)rax; + zf6 = 0x40; + r11 = (word32)((word32)rax); + if ((zf5) < (zf6)) { + goto L_AES_XTS_encrypt_aesni_done_64; + } + r11 = (word32)((word32)r11 & 0xffffffc0); +L_AES_XTS_encrypt_aesni_enc_64: + /* 64 bytes of input */ + /* aes_enc_64 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = x0; + x1 = x0; + x4 = _mm_srai_epi32(x4, 31); + x1 = _mm_slli_epi32(x1, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x1 = _mm_xor_si128(x1, x4); + x4 = x1; + x2 = x1; + x4 = _mm_srai_epi32(x4, 31); + x2 = _mm_slli_epi32(x2, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x2 = _mm_xor_si128(x2, x4); + x4 = x2; + x3 = x2; + x4 = _mm_srai_epi32(x4, 31); + x3 = _mm_slli_epi32(x3, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x3 = _mm_xor_si128(x3, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_xor_si128(x8, x4); + x9 = _mm_xor_si128(x9, x4); + x10 = _mm_xor_si128(x10, x4); + x11 = _mm_xor_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + zf7 = (word32)r10; + zf8 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_encrypt_aesni_aes_enc_64_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + zf9 = (word32)r10; + zf10 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_encrypt_aesni_aes_enc_64_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_encrypt_aesni_aes_enc_64_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x4); + x9 = _mm_aesenclast_si128(x9, x4); + x10 = _mm_aesenclast_si128(x10, x4); + x11 = _mm_aesenclast_si128(x11, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x4 = x3; + x0 = x3; + x4 = _mm_srai_epi32(x4, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r13 = (word32)((word32)r13 + 0x40); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_encrypt_aesni_enc_64; + } +L_AES_XTS_encrypt_aesni_done_64: + zf11 = (word32)r13; + zf12 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf11) == (zf12)) { + goto L_AES_XTS_encrypt_aesni_done_enc; + } + r11 = (word32)((word32)r11 - (word32)r13); + zf13 = (word32)r11; + zf14 = 0x10; + r11 = (word32)((word32)rax); + if ((zf13) < (zf14)) { + goto L_AES_XTS_encrypt_aesni_last_15; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + /* 16 bytes of input */ +L_AES_XTS_encrypt_aesni_enc_16: + rcx = (word64)(rdi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_enc_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesenc_si128(x8, x5); + zf15 = (word32)r10; + zf16 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_encrypt_aesni_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesenc_si128(x8, x6); + zf17 = (word32)r10; + zf18 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_encrypt_aesni_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesenc_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_encrypt_aesni_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); + x4 = x0; + x4 = _mm_srai_epi32(x4, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r13 = (word32)((word32)r13 + 0x10); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_encrypt_aesni_enc_16; + } + if (((word32)r13) == ((word32)rax)) { + goto L_AES_XTS_encrypt_aesni_done_enc; + } +L_AES_XTS_encrypt_aesni_last_15: + r13 = (word64)(r13 - 0x10); + rcx = (word64)(rsi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + r13 = (word64)(r13 + 0x10); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + rdx = (word64)(0); +L_AES_XTS_encrypt_aesni_last_15_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r13)) & 0xff); + WC_S8(rsi, r13) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r13 = (word32)((word32)r13 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r13) < ((word32)rax)) { + goto L_AES_XTS_encrypt_aesni_last_15_byte_loop; + } + r13 = (word64)(r13 - rdx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + r13 = (word64)(r13 - 0x10); + x8 = _mm_xor_si128(x8, x0); + /* aes_enc_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesenc_si128(x8, x5); + zf19 = (word32)r10; + zf20 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_encrypt_aesni_last_15_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesenc_si128(x8, x6); + zf21 = (word32)r10; + zf22 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_XTS_encrypt_aesni_last_15_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesenc_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_encrypt_aesni_last_15_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); +L_AES_XTS_encrypt_aesni_done_enc: + ; +} + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_XTS_encrypt_update_aesni(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr) +{ + word64 rdi, rsi, rax, r10, r8, r9, rsp, r12, r11, rcx = 0, rdx = 0; + __m128i x0, x1 = _mm_setzero_si128(), x2 = _mm_setzero_si128(), + x3 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), x12; + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r10 = (word64)(size_t)key; + r8 = (word64)(size_t)tweak_block; + r9 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_xts_gc_xts, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + r12 = (word32)(0); + zf1 = (word32)rax; + zf2 = 0x40; + r11 = (word32)((word32)rax); + if ((zf1) < (zf2)) { + goto L_AES_XTS_encrypt_update_aesni_done_64; + } + r11 = (word32)((word32)r11 & 0xffffffc0); +L_AES_XTS_encrypt_update_aesni_enc_64: + /* 64 bytes of input */ + /* aes_enc_64 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = x0; + x1 = x0; + x4 = _mm_srai_epi32(x4, 31); + x1 = _mm_slli_epi32(x1, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x1 = _mm_xor_si128(x1, x4); + x4 = x1; + x2 = x1; + x4 = _mm_srai_epi32(x4, 31); + x2 = _mm_slli_epi32(x2, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x2 = _mm_xor_si128(x2, x4); + x4 = x2; + x3 = x2; + x4 = _mm_srai_epi32(x4, 31); + x3 = _mm_slli_epi32(x3, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x3 = _mm_xor_si128(x3, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x8 = _mm_xor_si128(x8, x4); + x9 = _mm_xor_si128(x9, x4); + x10 = _mm_xor_si128(x10, x4); + x11 = _mm_xor_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + zf3 = (word32)r9; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_encrypt_update_aesni_aes_enc_64_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + zf5 = (word32)r9; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_XTS_encrypt_update_aesni_aes_enc_64_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_encrypt_update_aesni_aes_enc_64_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x4); + x9 = _mm_aesenclast_si128(x9, x4); + x10 = _mm_aesenclast_si128(x10, x4); + x11 = _mm_aesenclast_si128(x11, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x4 = x3; + x0 = x3; + x4 = _mm_srai_epi32(x4, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r12 = (word32)((word32)r12 + 0x40); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_encrypt_update_aesni_enc_64; + } +L_AES_XTS_encrypt_update_aesni_done_64: + zf7 = (word32)r12; + zf8 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf7) == (zf8)) { + goto L_AES_XTS_encrypt_update_aesni_done_enc; + } + r11 = (word32)((word32)r11 - (word32)r12); + zf9 = (word32)r11; + zf10 = 0x10; + r11 = (word32)((word32)rax); + if ((zf9) < (zf10)) { + goto L_AES_XTS_encrypt_update_aesni_last_15; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + /* 16 bytes of input */ +L_AES_XTS_encrypt_update_aesni_enc_16: + rcx = (word64)(rdi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_enc_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesenc_si128(x8, x5); + zf11 = (word32)r9; + zf12 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_encrypt_update_aesni_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesenc_si128(x8, x6); + zf13 = (word32)r9; + zf14 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_encrypt_update_aesni_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesenc_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_encrypt_update_aesni_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); + x4 = x0; + x4 = _mm_srai_epi32(x4, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r12 = (word32)((word32)r12 + 0x10); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_encrypt_update_aesni_enc_16; + } + if (((word32)r12) == ((word32)rax)) { + goto L_AES_XTS_encrypt_update_aesni_done_enc; + } +L_AES_XTS_encrypt_update_aesni_last_15: + r12 = (word64)(r12 - 0x10); + rcx = (word64)(rsi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + r12 = (word64)(r12 + 0x10); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + rdx = (word64)(0); +L_AES_XTS_encrypt_update_aesni_last_15_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r12)) & 0xff); + WC_S8(rsi, r12) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r12 = (word32)((word32)r12 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r12) < ((word32)rax)) { + goto L_AES_XTS_encrypt_update_aesni_last_15_byte_loop; + } + r12 = (word64)(r12 - rdx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + r12 = (word64)(r12 - 0x10); + x8 = _mm_xor_si128(x8, x0); + /* aes_enc_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesenc_si128(x8, x5); + zf15 = (word32)r9; + zf16 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_encrypt_update_aesni_last_15_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesenc_si128(x8, x6); + zf17 = (word32)r9; + zf18 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_encrypt_update_aesni_last_15_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesenc_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_encrypt_update_aesni_last_15_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); +L_AES_XTS_encrypt_update_aesni_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x0); +} + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_XTS_decrypt_aesni(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr) +{ + word64 rdi, rsi, rax, r12, r8, r9, r10, rsp, r13 = 0, r11 = 0, rcx = 0, + rdx = 0; + __m128i x0, x1 = _mm_setzero_si128(), x2 = _mm_setzero_si128(), + x3 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), x5, + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), x12; + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r12 = (word64)(size_t)i; + r8 = (word64)(size_t)key; + r9 = (word64)(size_t)key2; + r10 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_xts_gc_xts, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf1 = (word32)r10; + zf2 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_decrypt_aesni_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf3 = (word32)r10; + zf4 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_decrypt_aesni_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 224)); +L_AES_XTS_decrypt_aesni_tweak_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r13 = (word32)(0); + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_aesni_mul16_64; + } + r11 = (word32)((word32)r11 - 0x10); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_aesni_last_31_start; + } +L_AES_XTS_decrypt_aesni_mul16_64: + if (((word32)r11) < (0x40)) { + goto L_AES_XTS_decrypt_aesni_done_64; + } + r11 = (word32)((word32)r11 & 0xffffffc0); +L_AES_XTS_decrypt_aesni_dec_64: + /* 64 bytes of input */ + /* aes_dec_64 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = x0; + x1 = x0; + x4 = _mm_srai_epi32(x4, 31); + x1 = _mm_slli_epi32(x1, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x1 = _mm_xor_si128(x1, x4); + x4 = x1; + x2 = x1; + x4 = _mm_srai_epi32(x4, 31); + x2 = _mm_slli_epi32(x2, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x2 = _mm_xor_si128(x2, x4); + x4 = x2; + x3 = x2; + x4 = _mm_srai_epi32(x4, 31); + x3 = _mm_slli_epi32(x3, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x3 = _mm_xor_si128(x3, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + /* aes_dec_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_xor_si128(x8, x4); + x9 = _mm_xor_si128(x9, x4); + x10 = _mm_xor_si128(x10, x4); + x11 = _mm_xor_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + zf5 = (word32)r10; + zf6 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_XTS_decrypt_aesni_aes_dec_64_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + zf7 = (word32)r10; + zf8 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_decrypt_aesni_aes_dec_64_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_decrypt_aesni_aes_dec_64_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x4); + x9 = _mm_aesdeclast_si128(x9, x4); + x10 = _mm_aesdeclast_si128(x10, x4); + x11 = _mm_aesdeclast_si128(x11, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x4 = x3; + x0 = x3; + x4 = _mm_srai_epi32(x4, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r13 = (word32)((word32)r13 + 0x40); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_decrypt_aesni_dec_64; + } +L_AES_XTS_decrypt_aesni_done_64: + zf9 = (word32)r13; + zf10 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf9) == (zf10)) { + goto L_AES_XTS_decrypt_aesni_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_aesni_mul16; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r13); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_aesni_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r13); +L_AES_XTS_decrypt_aesni_mul16: +L_AES_XTS_decrypt_aesni_dec_16: + /* 16 bytes of input */ + rcx = (word64)(rdi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf11 = (word32)r10; + zf12 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_decrypt_aesni_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf13 = (word32)r10; + zf14 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_decrypt_aesni_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_decrypt_aesni_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); + x4 = x0; + x4 = _mm_srai_epi32(x4, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r13 = (word32)((word32)r13 + 0x10); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_decrypt_aesni_dec_16; + } + if (((word32)r13) == ((word32)rax)) { + goto L_AES_XTS_decrypt_aesni_done_dec; + } +L_AES_XTS_decrypt_aesni_last_31_start: + x4 = x0; + x7 = x0; + x4 = _mm_srai_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x7 = _mm_xor_si128(x7, x4); + rcx = (word64)(rdi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x7); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf15 = (word32)r10; + zf16 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_decrypt_aesni_last_31_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf17 = (word32)r10; + zf18 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_decrypt_aesni_last_31_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_decrypt_aesni_last_31_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + r13 = (word64)(r13 + 0x10); + rdx = (word64)(0); +L_AES_XTS_decrypt_aesni_last_31_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r13)) & 0xff); + WC_S8(rsi, r13) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r13 = (word32)((word32)r13 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r13) < ((word32)rax)) { + goto L_AES_XTS_decrypt_aesni_last_31_byte_loop; + } + r13 = (word64)(r13 - rdx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf19 = (word32)r10; + zf20 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_decrypt_aesni_last_31_2_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf21 = (word32)r10; + zf22 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_XTS_decrypt_aesni_last_31_2_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_decrypt_aesni_last_31_2_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + r13 = (word64)(r13 - 0x10); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); +L_AES_XTS_decrypt_aesni_done_dec: + ; +} + +WC_X64I_TARGET("sse2,aes") +WOLFSSL_LOCAL void AES_XTS_decrypt_update_aesni(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr) +{ + word64 rdi, rsi, rax, r10, r8, r9, rsp, r12, r11, rcx = 0, rdx = 0; + __m128i x0, x1 = _mm_setzero_si128(), x2 = _mm_setzero_si128(), + x3 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12; + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r10 = (word64)(size_t)key; + r8 = (word64)(size_t)tweak_block; + r9 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_aes_xts_gc_xts, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + r12 = (word32)(0); + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_aesni_mul16_64; + } + r11 = (word32)((word32)r11 - 0x10); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_aesni_last_31_start; + } +L_AES_XTS_decrypt_update_aesni_mul16_64: + if (((word32)r11) < (0x40)) { + goto L_AES_XTS_decrypt_update_aesni_done_64; + } + r11 = (word32)((word32)r11 & 0xffffffc0); +L_AES_XTS_decrypt_update_aesni_dec_64: + /* 64 bytes of input */ + /* aes_dec_64 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = x0; + x1 = x0; + x4 = _mm_srai_epi32(x4, 31); + x1 = _mm_slli_epi32(x1, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x1 = _mm_xor_si128(x1, x4); + x4 = x1; + x2 = x1; + x4 = _mm_srai_epi32(x4, 31); + x2 = _mm_slli_epi32(x2, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x2 = _mm_xor_si128(x2, x4); + x4 = x2; + x3 = x2; + x4 = _mm_srai_epi32(x4, 31); + x3 = _mm_slli_epi32(x3, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x3 = _mm_xor_si128(x3, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + /* aes_dec_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x8 = _mm_xor_si128(x8, x4); + x9 = _mm_xor_si128(x9, x4); + x10 = _mm_xor_si128(x10, x4); + x11 = _mm_xor_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + zf1 = (word32)r9; + zf2 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_decrypt_update_aesni_aes_dec_64_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + zf3 = (word32)r9; + zf4 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_decrypt_update_aesni_aes_dec_64_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_decrypt_update_aesni_aes_dec_64_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x4); + x9 = _mm_aesdeclast_si128(x9, x4); + x10 = _mm_aesdeclast_si128(x10, x4); + x11 = _mm_aesdeclast_si128(x11, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x4 = x3; + x0 = x3; + x4 = _mm_srai_epi32(x4, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r12 = (word32)((word32)r12 + 0x40); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_decrypt_update_aesni_dec_64; + } +L_AES_XTS_decrypt_update_aesni_done_64: + zf5 = (word32)r12; + zf6 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf5) == (zf6)) { + goto L_AES_XTS_decrypt_update_aesni_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_aesni_mul16; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r12); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_aesni_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r12); +L_AES_XTS_decrypt_update_aesni_mul16: +L_AES_XTS_decrypt_update_aesni_dec_16: + /* 16 bytes of input */ + rcx = (word64)(rdi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf7 = (word32)r9; + zf8 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_decrypt_update_aesni_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf9 = (word32)r9; + zf10 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_decrypt_update_aesni_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_decrypt_update_aesni_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); + x4 = x0; + x4 = _mm_srai_epi32(x4, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r12 = (word32)((word32)r12 + 0x10); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_decrypt_update_aesni_dec_16; + } + if (((word32)r12) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_aesni_done_dec; + } +L_AES_XTS_decrypt_update_aesni_last_31_start: + x4 = x0; + x7 = x0; + x4 = _mm_srai_epi32(x4, 31); + x7 = _mm_slli_epi32(x7, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x7 = _mm_xor_si128(x7, x4); + rcx = (word64)(rdi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x7); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf11 = (word32)r9; + zf12 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_decrypt_update_aesni_last_31_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf13 = (word32)r9; + zf14 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_decrypt_update_aesni_last_31_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_decrypt_update_aesni_last_31_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + r12 = (word64)(r12 + 0x10); + rdx = (word64)(0); +L_AES_XTS_decrypt_update_aesni_last_31_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r12)) & 0xff); + WC_S8(rsi, r12) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r12 = (word32)((word32)r12 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r12) < ((word32)rax)) { + goto L_AES_XTS_decrypt_update_aesni_last_31_byte_loop; + } + r12 = (word64)(r12 - rdx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf15 = (word32)r9; + zf16 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_decrypt_update_aesni_last_31_2_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf17 = (word32)r9; + zf18 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_decrypt_update_aesni_last_31_2_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_decrypt_update_aesni_last_31_2_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + r12 = (word64)(r12 - 0x10); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); +L_AES_XTS_decrypt_update_aesni_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x0); +} + +#ifdef HAVE_INTEL_AVX1 +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_XTS_init_avx1(const unsigned char* i, + const unsigned char* key, int nr) +{ + word64 rdi, rsi, rdx; + __m128i x0, x2, x3 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)i; + rsi = (word64)(size_t)key; + rdx = (word64)(word32)nr; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x0 = _mm_aesenc_si128(x0, x2); + zf1 = (word32)rdx; + zf2 = 0xb; + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_init_avx1_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x2); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_aesenc_si128(x0, x3); + zf3 = (word32)rdx; + zf4 = 0xd; + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_init_avx1_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x2); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x0 = _mm_aesenc_si128(x0, x3); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_XTS_init_avx1_tweak_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); +} + +XALIGNED(16) static const word32 L_avx1_aes_xts_gc_xts[] WC_X64I_UNUSED = { + 0x00000087, 0x00000001, 0x00000001, 0x00000001, +}; + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_XTS_encrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr) +{ + word64 rdi, rsi, rax, r12, r8, r9, r10, rsp, r13 = 0, r11 = 0, rcx = 0, + rdx = 0; + __m128i x0, x1 = _mm_setzero_si128(), x2 = _mm_setzero_si128(), + x3 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), x5, + x6 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12; + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r12 = (word64)(size_t)i; + r8 = (word64)(size_t)key; + r9 = (word64)(size_t)key2; + r10 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_xts_gc_xts, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf1 = (word32)r10; + zf2 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_encrypt_avx1_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf3 = (word32)r10; + zf4 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_encrypt_avx1_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 224)); +L_AES_XTS_encrypt_avx1_tweak_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r13 = (word32)(0); + zf5 = (word32)rax; + zf6 = 0x40; + r11 = (word32)((word32)rax); + if ((zf5) < (zf6)) { + goto L_AES_XTS_encrypt_avx1_done_64; + } + r11 = (word32)((word32)r11 & 0xffffffc0); +L_AES_XTS_encrypt_avx1_enc_64: + /* 64 bytes of input */ + /* aes_enc_64 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = _mm_srai_epi32(x0, 31); + x1 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x1 = _mm_xor_si128(x1, x4); + x4 = _mm_srai_epi32(x1, 31); + x2 = _mm_slli_epi32(x1, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x2 = _mm_xor_si128(x2, x4); + x4 = _mm_srai_epi32(x2, 31); + x3 = _mm_slli_epi32(x2, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x3 = _mm_xor_si128(x3, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_xor_si128(x8, x4); + x9 = _mm_xor_si128(x9, x4); + x10 = _mm_xor_si128(x10, x4); + x11 = _mm_xor_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + zf7 = (word32)r10; + zf8 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_encrypt_avx1_aes_enc_64_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + zf9 = (word32)r10; + zf10 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_encrypt_avx1_aes_enc_64_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_encrypt_avx1_aes_enc_64_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x4); + x9 = _mm_aesenclast_si128(x9, x4); + x10 = _mm_aesenclast_si128(x10, x4); + x11 = _mm_aesenclast_si128(x11, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x4 = _mm_srai_epi32(x3, 31); + x0 = _mm_slli_epi32(x3, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r13 = (word32)((word32)r13 + 0x40); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_encrypt_avx1_enc_64; + } +L_AES_XTS_encrypt_avx1_done_64: + zf11 = (word32)r13; + zf12 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf11) == (zf12)) { + goto L_AES_XTS_encrypt_avx1_done_enc; + } + r11 = (word32)((word32)r11 - (word32)r13); + zf13 = (word32)r11; + zf14 = 0x10; + r11 = (word32)((word32)rax); + if ((zf13) < (zf14)) { + goto L_AES_XTS_encrypt_avx1_last_15; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + /* 16 bytes of input */ +L_AES_XTS_encrypt_avx1_enc_16: + rcx = (word64)(rdi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_enc_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesenc_si128(x8, x5); + zf15 = (word32)r10; + zf16 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_encrypt_avx1_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesenc_si128(x8, x6); + zf17 = (word32)r10; + zf18 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_encrypt_avx1_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesenc_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_encrypt_avx1_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); + x4 = _mm_srai_epi32(x0, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r13 = (word32)((word32)r13 + 0x10); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_encrypt_avx1_enc_16; + } + if (((word32)r13) == ((word32)rax)) { + goto L_AES_XTS_encrypt_avx1_done_enc; + } +L_AES_XTS_encrypt_avx1_last_15: + r13 = (word64)(r13 - 0x10); + rcx = (word64)(rsi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + r13 = (word64)(r13 + 0x10); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + rdx = (word64)(0); +L_AES_XTS_encrypt_avx1_last_15_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r13)) & 0xff); + WC_S8(rsi, r13) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r13 = (word32)((word32)r13 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r13) < ((word32)rax)) { + goto L_AES_XTS_encrypt_avx1_last_15_byte_loop; + } + r13 = (word64)(r13 - rdx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + r13 = (word64)(r13 - 0x10); + x8 = _mm_xor_si128(x8, x0); + /* aes_enc_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesenc_si128(x8, x5); + zf19 = (word32)r10; + zf20 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_encrypt_avx1_last_15_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesenc_si128(x8, x6); + zf21 = (word32)r10; + zf22 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_XTS_encrypt_avx1_last_15_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesenc_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_encrypt_avx1_last_15_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); +L_AES_XTS_encrypt_avx1_done_enc: + ; +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_XTS_encrypt_update_avx1(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr) +{ + word64 rdi, rsi, rax, r10, r8, r9, rsp, r12, r11, rcx = 0, rdx = 0; + __m128i x0, x1 = _mm_setzero_si128(), x2 = _mm_setzero_si128(), + x3 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), x12; + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r10 = (word64)(size_t)key; + r8 = (word64)(size_t)tweak_block; + r9 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_xts_gc_xts, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + r12 = (word32)(0); + zf1 = (word32)rax; + zf2 = 0x40; + r11 = (word32)((word32)rax); + if ((zf1) < (zf2)) { + goto L_AES_XTS_encrypt_update_avx1_done_64; + } + r11 = (word32)((word32)r11 & 0xffffffc0); +L_AES_XTS_encrypt_update_avx1_enc_64: + /* 64 bytes of input */ + /* aes_enc_64 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = _mm_srai_epi32(x0, 31); + x1 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x1 = _mm_xor_si128(x1, x4); + x4 = _mm_srai_epi32(x1, 31); + x2 = _mm_slli_epi32(x1, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x2 = _mm_xor_si128(x2, x4); + x4 = _mm_srai_epi32(x2, 31); + x3 = _mm_slli_epi32(x2, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x3 = _mm_xor_si128(x3, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + /* aes_enc_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x8 = _mm_xor_si128(x8, x4); + x9 = _mm_xor_si128(x9, x4); + x10 = _mm_xor_si128(x10, x4); + x11 = _mm_xor_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + zf3 = (word32)r9; + zf4 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_encrypt_update_avx1_aes_enc_64_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + zf5 = (word32)r9; + zf6 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_XTS_encrypt_update_avx1_aes_enc_64_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesenc_si128(x8, x4); + x9 = _mm_aesenc_si128(x9, x4); + x10 = _mm_aesenc_si128(x10, x4); + x11 = _mm_aesenc_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_encrypt_update_avx1_aes_enc_64_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x4); + x9 = _mm_aesenclast_si128(x9, x4); + x10 = _mm_aesenclast_si128(x10, x4); + x11 = _mm_aesenclast_si128(x11, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x4 = _mm_srai_epi32(x3, 31); + x0 = _mm_slli_epi32(x3, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r12 = (word32)((word32)r12 + 0x40); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_encrypt_update_avx1_enc_64; + } +L_AES_XTS_encrypt_update_avx1_done_64: + zf7 = (word32)r12; + zf8 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf7) == (zf8)) { + goto L_AES_XTS_encrypt_update_avx1_done_enc; + } + r11 = (word32)((word32)r11 - (word32)r12); + zf9 = (word32)r11; + zf10 = 0x10; + r11 = (word32)((word32)rax); + if ((zf9) < (zf10)) { + goto L_AES_XTS_encrypt_update_avx1_last_15; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + /* 16 bytes of input */ +L_AES_XTS_encrypt_update_avx1_enc_16: + rcx = (word64)(rdi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_enc_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesenc_si128(x8, x5); + zf11 = (word32)r9; + zf12 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_encrypt_update_avx1_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesenc_si128(x8, x6); + zf13 = (word32)r9; + zf14 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_encrypt_update_avx1_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesenc_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_encrypt_update_avx1_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); + x4 = _mm_srai_epi32(x0, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r12 = (word32)((word32)r12 + 0x10); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_encrypt_update_avx1_enc_16; + } + if (((word32)r12) == ((word32)rax)) { + goto L_AES_XTS_encrypt_update_avx1_done_enc; + } +L_AES_XTS_encrypt_update_avx1_last_15: + r12 = (word64)(r12 - 0x10); + rcx = (word64)(rsi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + r12 = (word64)(r12 + 0x10); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + rdx = (word64)(0); +L_AES_XTS_encrypt_update_avx1_last_15_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r12)) & 0xff); + WC_S8(rsi, r12) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r12 = (word32)((word32)r12 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r12) < ((word32)rax)) { + goto L_AES_XTS_encrypt_update_avx1_last_15_byte_loop; + } + r12 = (word64)(r12 - rdx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + r12 = (word64)(r12 - 0x10); + x8 = _mm_xor_si128(x8, x0); + /* aes_enc_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesenc_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesenc_si128(x8, x5); + zf15 = (word32)r9; + zf16 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_encrypt_update_avx1_last_15_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesenc_si128(x8, x6); + zf17 = (word32)r9; + zf18 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_encrypt_update_avx1_last_15_aes_enc_block_last; + } + x8 = _mm_aesenc_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesenc_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_encrypt_update_avx1_last_15_aes_enc_block_last: + x8 = _mm_aesenclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); +L_AES_XTS_encrypt_update_avx1_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x0); +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_XTS_decrypt_avx1(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr) +{ + word64 rdi, rsi, rax, r12, r8, r9, r10, rsp, r13 = 0, r11 = 0, rcx = 0, + rdx = 0; + __m128i x0, x1 = _mm_setzero_si128(), x2 = _mm_setzero_si128(), + x3 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), x5, + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), x12; + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r12 = (word64)(size_t)i; + r8 = (word64)(size_t)key; + r9 = (word64)(size_t)key2; + r10 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_xts_gc_xts, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r12, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 16)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 32)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 48)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 64)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 80)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 96)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 112)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 128)); + x0 = _mm_aesenc_si128(x0, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 144)); + x0 = _mm_aesenc_si128(x0, x5); + zf1 = (word32)r10; + zf2 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_decrypt_avx1_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 176)); + x0 = _mm_aesenc_si128(x0, x6); + zf3 = (word32)r10; + zf4 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_decrypt_avx1_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 208)); + x0 = _mm_aesenc_si128(x0, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 224)); +L_AES_XTS_decrypt_avx1_tweak_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x5); + r13 = (word32)(0); + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_avx1_mul16_64; + } + r11 = (word32)((word32)r11 - 0x10); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_avx1_last_31_start; + } +L_AES_XTS_decrypt_avx1_mul16_64: + if (((word32)r11) < (0x40)) { + goto L_AES_XTS_decrypt_avx1_done_64; + } + r11 = (word32)((word32)r11 & 0xffffffc0); +L_AES_XTS_decrypt_avx1_dec_64: + /* 64 bytes of input */ + /* aes_dec_64 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = _mm_srai_epi32(x0, 31); + x1 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x1 = _mm_xor_si128(x1, x4); + x4 = _mm_srai_epi32(x1, 31); + x2 = _mm_slli_epi32(x1, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x2 = _mm_xor_si128(x2, x4); + x4 = _mm_srai_epi32(x2, 31); + x3 = _mm_slli_epi32(x2, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x3 = _mm_xor_si128(x3, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + /* aes_dec_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + x8 = _mm_xor_si128(x8, x4); + x9 = _mm_xor_si128(x9, x4); + x10 = _mm_xor_si128(x10, x4); + x11 = _mm_xor_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + zf5 = (word32)r10; + zf6 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_XTS_decrypt_avx1_aes_dec_64_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + zf7 = (word32)r10; + zf8 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_decrypt_avx1_aes_dec_64_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_decrypt_avx1_aes_dec_64_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x4); + x9 = _mm_aesdeclast_si128(x9, x4); + x10 = _mm_aesdeclast_si128(x10, x4); + x11 = _mm_aesdeclast_si128(x11, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x4 = _mm_srai_epi32(x3, 31); + x0 = _mm_slli_epi32(x3, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r13 = (word32)((word32)r13 + 0x40); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_decrypt_avx1_dec_64; + } +L_AES_XTS_decrypt_avx1_done_64: + zf9 = (word32)r13; + zf10 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf9) == (zf10)) { + goto L_AES_XTS_decrypt_avx1_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_avx1_mul16; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r13); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_avx1_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r13); +L_AES_XTS_decrypt_avx1_mul16: +L_AES_XTS_decrypt_avx1_dec_16: + /* 16 bytes of input */ + rcx = (word64)(rdi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf11 = (word32)r10; + zf12 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_decrypt_avx1_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf13 = (word32)r10; + zf14 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_decrypt_avx1_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_decrypt_avx1_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); + x4 = _mm_srai_epi32(x0, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r13 = (word32)((word32)r13 + 0x10); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_decrypt_avx1_dec_16; + } + if (((word32)r13) == ((word32)rax)) { + goto L_AES_XTS_decrypt_avx1_done_dec; + } +L_AES_XTS_decrypt_avx1_last_31_start: + x4 = _mm_srai_epi32(x0, 31); + x7 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x7 = _mm_xor_si128(x7, x4); + rcx = (word64)(rdi + r13); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x7); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf15 = (word32)r10; + zf16 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_decrypt_avx1_last_31_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf17 = (word32)r10; + zf18 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_decrypt_avx1_last_31_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_decrypt_avx1_last_31_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + r13 = (word64)(r13 + 0x10); + rdx = (word64)(0); +L_AES_XTS_decrypt_avx1_last_31_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r13)) & 0xff); + WC_S8(rsi, r13) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r13 = (word32)((word32)r13 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r13) < ((word32)rax)) { + goto L_AES_XTS_decrypt_avx1_last_31_byte_loop; + } + r13 = (word64)(r13 - rdx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf19 = (word32)r10; + zf20 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 160)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_decrypt_avx1_last_31_2_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf21 = (word32)r10; + zf22 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 192)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_XTS_decrypt_avx1_last_31_2_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 224)); +L_AES_XTS_decrypt_avx1_last_31_2_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + r13 = (word64)(r13 - 0x10); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); +L_AES_XTS_decrypt_avx1_done_dec: + ; +} + +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_XTS_decrypt_update_avx1(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr) +{ + word64 rdi, rsi, rax, r10, r8, r9, rsp, r12, r11, rcx = 0, rdx = 0; + __m128i x0, x1 = _mm_setzero_si128(), x2 = _mm_setzero_si128(), + x3 = _mm_setzero_si128(), x4 = _mm_setzero_si128(), + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12; + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r10 = (word64)(size_t)key; + r8 = (word64)(size_t)tweak_block; + r9 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_aes_xts_gc_xts, 0)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)); + r12 = (word32)(0); + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx1_mul16_64; + } + r11 = (word32)((word32)r11 - 0x10); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_avx1_last_31_start; + } +L_AES_XTS_decrypt_update_avx1_mul16_64: + if (((word32)r11) < (0x40)) { + goto L_AES_XTS_decrypt_update_avx1_done_64; + } + r11 = (word32)((word32)r11 & 0xffffffc0); +L_AES_XTS_decrypt_update_avx1_dec_64: + /* 64 bytes of input */ + /* aes_dec_64 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48)); + x4 = _mm_srai_epi32(x0, 31); + x1 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x1 = _mm_xor_si128(x1, x4); + x4 = _mm_srai_epi32(x1, 31); + x2 = _mm_slli_epi32(x1, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x2 = _mm_xor_si128(x2, x4); + x4 = _mm_srai_epi32(x2, 31); + x3 = _mm_slli_epi32(x2, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x3 = _mm_xor_si128(x3, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + /* aes_dec_block */ + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x8 = _mm_xor_si128(x8, x4); + x9 = _mm_xor_si128(x9, x4); + x10 = _mm_xor_si128(x10, x4); + x11 = _mm_xor_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + zf1 = (word32)r9; + zf2 = 0xb; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_decrypt_update_avx1_aes_dec_64_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + zf3 = (word32)r9; + zf4 = 0xd; + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_decrypt_update_avx1_aes_dec_64_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesdec_si128(x8, x4); + x9 = _mm_aesdec_si128(x9, x4); + x10 = _mm_aesdec_si128(x10, x4); + x11 = _mm_aesdec_si128(x11, x4); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_decrypt_update_avx1_aes_dec_64_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x4); + x9 = _mm_aesdeclast_si128(x9, x4); + x10 = _mm_aesdeclast_si128(x10, x4); + x11 = _mm_aesdeclast_si128(x11, x4); + x8 = _mm_xor_si128(x8, x0); + x9 = _mm_xor_si128(x9, x1); + x10 = _mm_xor_si128(x10, x2); + x11 = _mm_xor_si128(x11, x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x11); + x4 = _mm_srai_epi32(x3, 31); + x0 = _mm_slli_epi32(x3, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r12 = (word32)((word32)r12 + 0x40); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_decrypt_update_avx1_dec_64; + } +L_AES_XTS_decrypt_update_avx1_done_64: + zf5 = (word32)r12; + zf6 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf5) == (zf6)) { + goto L_AES_XTS_decrypt_update_avx1_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx1_mul16; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r12); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_avx1_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r12); +L_AES_XTS_decrypt_update_avx1_mul16: +L_AES_XTS_decrypt_update_avx1_dec_16: + /* 16 bytes of input */ + rcx = (word64)(rdi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf7 = (word32)r9; + zf8 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_decrypt_update_avx1_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf9 = (word32)r9; + zf10 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_decrypt_update_avx1_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_decrypt_update_avx1_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); + x4 = _mm_srai_epi32(x0, 31); + x0 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x0 = _mm_xor_si128(x0, x4); + r12 = (word32)((word32)r12 + 0x10); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_decrypt_update_avx1_dec_16; + } + if (((word32)r12) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx1_done_dec; + } +L_AES_XTS_decrypt_update_avx1_last_31_start: + x4 = _mm_srai_epi32(x0, 31); + x7 = _mm_slli_epi32(x0, 1); + x4 = _mm_shuffle_epi32(x4, 0x93); + x4 = _mm_and_si128(x4, x12); + x7 = _mm_xor_si128(x7, x4); + rcx = (word64)(rdi + r12); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0)); + x8 = _mm_xor_si128(x8, x7); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf11 = (word32)r9; + zf12 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_decrypt_update_avx1_last_31_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf13 = (word32)r9; + zf14 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_decrypt_update_avx1_last_31_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_decrypt_update_avx1_last_31_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x7); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + r12 = (word64)(r12 + 0x10); + rdx = (word64)(0); +L_AES_XTS_decrypt_update_avx1_last_31_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r12)) & 0xff); + WC_S8(rsi, r12) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r12 = (word32)((word32)r12 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r12) < ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx1_last_31_byte_loop; + } + r12 = (word64)(r12 - rdx); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsp, 0)); + x8 = _mm_xor_si128(x8, x0); + /* aes_dec_block */ + x8 = _mm_xor_si128(x8, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 128)); + x8 = _mm_aesdec_si128(x8, x5); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 144)); + x8 = _mm_aesdec_si128(x8, x5); + zf15 = (word32)r9; + zf16 = 0xb; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 160)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_decrypt_update_avx1_last_31_2_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 176)); + x8 = _mm_aesdec_si128(x8, x6); + zf17 = (word32)r9; + zf18 = 0xd; + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 192)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_decrypt_update_avx1_last_31_2_aes_dec_block_last; + } + x8 = _mm_aesdec_si128(x8, x5); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 208)); + x8 = _mm_aesdec_si128(x8, x6); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 224)); +L_AES_XTS_decrypt_update_avx1_last_31_2_aes_dec_block_last: + x8 = _mm_aesdeclast_si128(x8, x5); + x8 = _mm_xor_si128(x8, x0); + r12 = (word64)(r12 - 0x10); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x8); +L_AES_XTS_decrypt_update_avx1_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x0); +} + +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_VAES +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_XTS_init_vaes(const unsigned char* i, + const unsigned char* key, int nr) +{ + word64 rdi, rsi, rdx; + __m128i x0, x2, x3 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)i; + rsi = (word64)(size_t)key; + rdx = (word64)(word32)nr; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x0 = _mm_aesenc_si128(x0, x2); + zf1 = (word32)rdx; + zf2 = 0xb; + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_init_vaes_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x2); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_aesenc_si128(x0, x3); + zf3 = (word32)rdx; + zf4 = 0xd; + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_init_vaes_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x2); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x0 = _mm_aesenc_si128(x0, x3); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_XTS_init_vaes_tweak_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); +} + +XALIGNED(16) static const word32 L_vaes_aes_xts_gc_xts[] WC_X64I_UNUSED = { + 0x00000087, 0x00000000, 0x00000001, 0x00000000, +}; + +XALIGNED(16) static const word32 L_vaes_aes_xts_poly[] WC_X64I_UNUSED = { + 0x00000087, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_vaes_aes_xts_shl[] WC_X64I_UNUSED = { + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000001, 0x00000000, 0x00000001, 0x00000000, +}; + +XALIGNED(16) static const word32 L_vaes_aes_xts_shr[] WC_X64I_UNUSED = { + 0x00000040, 0x00000000, 0x00000040, 0x00000000, + 0x0000003f, 0x00000000, 0x0000003f, 0x00000000, +}; + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_XTS_encrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr) +{ + word64 rdi, rsi, rax, r12, r8, r9, r10, rsp, r13 = 0, r11 = 0, rcx = 0, + rdx = 0; + __m128i x12; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5, y6 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(), y8, y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y13, y14, y15; + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r12 = (word64)(size_t)i; + r8 = (word64)(size_t)key; + r9 = (word64)(size_t)key2; + r10 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_xts_gc_xts, 0)); + y13 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_xts_poly, 0))); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(L_vaes_aes_xts_shl, 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(L_vaes_aes_xts_shr, 0)); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + /* aes_enc_block */ + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 16))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 32))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 48))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 64))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 80))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 96))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 112))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 128))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 144))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + zf1 = (word32)r10; + zf2 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_encrypt_vaes_tweak_aes_enc_block_last; + } + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 176))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y6))); + zf3 = (word32)r10; + zf4 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_encrypt_vaes_tweak_aes_enc_block_last; + } + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 208))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 224))); +L_AES_XTS_encrypt_vaes_tweak_aes_enc_block_last: + y8 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + r13 = (word32)(0); + if (((word32)rax) < (0x20)) { + goto L_AES_XTS_encrypt_vaes_done_128; + } + zf5 = (word32)rax; + zf6 = 0x80; + r11 = (word32)((word32)rax); + if ((zf5) < (zf6)) { + goto L_AES_XTS_encrypt_vaes_done_128; + } + r11 = (word32)((word32)r11 & 0xffffff80); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + y9 = _mm256_srli_epi64(y4, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y4, 2); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + y9 = _mm256_srli_epi64(y5, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_slli_epi64(y5, 2); + y6 = _mm256_xor_si256(y6, y10); + y6 = _mm256_xor_si256(y6, y9); + y9 = _mm256_srli_epi64(y6, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_slli_epi64(y6, 2); + y7 = _mm256_xor_si256(y7, y10); + y7 = _mm256_xor_si256(y7, y9); +L_AES_XTS_encrypt_vaes_enc_128: + /* 128 bytes of input */ + /* aes_enc_128 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + /* aes_enc_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y5); + y1 = _mm256_xor_si256(y1, y9); + y2 = _mm256_xor_si256(y2, y6); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y7); + y3 = _mm256_xor_si256(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + zf7 = (word32)r10; + zf8 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_encrypt_vaes_aes_enc_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + zf9 = (word32)r10; + zf10 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_encrypt_vaes_aes_enc_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_encrypt_vaes_aes_enc_128_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y9); + y1 = _mm256_aesenclast_epi128(y1, y9); + y2 = _mm256_aesenclast_epi128(y2, y9); + y3 = _mm256_aesenclast_epi128(y3, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y9 = _mm256_srli_epi64(y4, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y4 = _mm256_slli_epi64(y4, 8); + y4 = _mm256_xor_si256(y4, y10); + y4 = _mm256_xor_si256(y4, y9); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y9 = _mm256_srli_epi64(y5, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y5, 8); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + y2 = _mm256_xor_si256(y2, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y9 = _mm256_srli_epi64(y6, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_slli_epi64(y6, 8); + y6 = _mm256_xor_si256(y6, y10); + y6 = _mm256_xor_si256(y6, y9); + y3 = _mm256_xor_si256(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + y9 = _mm256_srli_epi64(y7, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_slli_epi64(y7, 8); + y7 = _mm256_xor_si256(y7, y10); + y7 = _mm256_xor_si256(y7, y9); + r13 = (word32)((word32)r13 + 0x80); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_encrypt_vaes_enc_128; + } + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 0)); +L_AES_XTS_encrypt_vaes_done_128: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffffc0); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_encrypt_vaes_done_64; + } + /* 64 bytes of input */ + /* aes_enc_64 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + y9 = _mm256_srli_epi64(y4, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y4, 2); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + /* aes_enc_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y5); + y1 = _mm256_xor_si256(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + zf11 = (word32)r10; + zf12 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_encrypt_vaes_aes_enc_64_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + zf13 = (word32)r10; + zf14 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_encrypt_vaes_aes_enc_64_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_encrypt_vaes_aes_enc_64_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y9); + y1 = _mm256_aesenclast_epi128(y1, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y9 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y9), 31)); + y9 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y9), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + r13 = (word32)((word32)r13 + 0x40); +L_AES_XTS_encrypt_vaes_done_64: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffffe0); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_encrypt_vaes_done_32; + } + /* 32 bytes of input */ + /* aes_enc_32 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + /* aes_enc_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_aesenc_epi128(y0, y9); + zf15 = (word32)r10; + zf16 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_encrypt_vaes_aes_enc_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_aesenc_epi128(y0, y9); + zf17 = (word32)r10; + zf18 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_encrypt_vaes_aes_enc_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_encrypt_vaes_aes_enc_32_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y9 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y9), 31)); + y9 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y9), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + r13 = (word32)((word32)r13 + 0x20); +L_AES_XTS_encrypt_vaes_done_32: + zf19 = (word32)r13; + zf20 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf19) == (zf20)) { + goto L_AES_XTS_encrypt_vaes_done_enc; + } + r11 = (word32)((word32)r11 - (word32)r13); + zf21 = (word32)r11; + zf22 = 0x10; + r11 = (word32)((word32)rax); + if ((zf21) < (zf22)) { + goto L_AES_XTS_encrypt_vaes_last_15; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + /* 16 bytes of input */ +L_AES_XTS_encrypt_vaes_enc_16: + rcx = (word64)(rdi + r13); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + /* aes_enc_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf23 = (word32)r10; + zf24 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_XTS_encrypt_vaes_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf25 = (word32)r10; + zf26 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_XTS_encrypt_vaes_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_encrypt_vaes_aes_enc_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y0)); + y4 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y4), 31)); + y4 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y4), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y4))); + r13 = (word32)((word32)r13 + 0x10); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_encrypt_vaes_enc_16; + } + if (((word32)r13) == ((word32)rax)) { + goto L_AES_XTS_encrypt_vaes_done_enc; + } +L_AES_XTS_encrypt_vaes_last_15: + r13 = (word64)(r13 - 0x10); + rcx = (word64)(rsi + r13); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + r13 = (word64)(r13 + 0x10); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(0); +L_AES_XTS_encrypt_vaes_last_15_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r13)) & 0xff); + WC_S8(rsi, r13) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r13 = (word32)((word32)r13 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r13) < ((word32)rax)) { + goto L_AES_XTS_encrypt_vaes_last_15_byte_loop; + } + r13 = (word64)(r13 - rdx); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + r13 = (word64)(r13 - 0x10); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + /* aes_enc_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf27 = (word32)r10; + zf28 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_XTS_encrypt_vaes_last_15_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf29 = (word32)r10; + zf30 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_XTS_encrypt_vaes_last_15_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_encrypt_vaes_last_15_aes_enc_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y0)); +L_AES_XTS_encrypt_vaes_done_enc: + ; +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_XTS_encrypt_update_vaes(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr) +{ + word64 rdi, rsi, rax, r10, r8, r9, rsp, r12, r11 = 0, rcx = 0, rdx = 0; + __m128i x12; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), y8, + y9 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(), y13, + y14, y15; + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r10 = (word64)(size_t)key; + r8 = (word64)(size_t)tweak_block; + r9 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_xts_gc_xts, 0)); + y13 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_xts_poly, 0))); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(L_vaes_aes_xts_shl, 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(L_vaes_aes_xts_shr, 0)); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + r12 = (word32)(0); + if (((word32)rax) < (0x20)) { + goto L_AES_XTS_encrypt_update_vaes_done_128; + } + zf1 = (word32)rax; + zf2 = 0x80; + r11 = (word32)((word32)rax); + if ((zf1) < (zf2)) { + goto L_AES_XTS_encrypt_update_vaes_done_128; + } + r11 = (word32)((word32)r11 & 0xffffff80); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + y9 = _mm256_srli_epi64(y4, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y4, 2); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + y9 = _mm256_srli_epi64(y5, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_slli_epi64(y5, 2); + y6 = _mm256_xor_si256(y6, y10); + y6 = _mm256_xor_si256(y6, y9); + y9 = _mm256_srli_epi64(y6, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_slli_epi64(y6, 2); + y7 = _mm256_xor_si256(y7, y10); + y7 = _mm256_xor_si256(y7, y9); +L_AES_XTS_encrypt_update_vaes_enc_128: + /* 128 bytes of input */ + /* aes_enc_128 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + /* aes_enc_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y5); + y1 = _mm256_xor_si256(y1, y9); + y2 = _mm256_xor_si256(y2, y6); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y7); + y3 = _mm256_xor_si256(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + zf3 = (word32)r9; + zf4 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_encrypt_update_vaes_aes_enc_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + zf5 = (word32)r9; + zf6 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_XTS_encrypt_update_vaes_aes_enc_128_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y2 = _mm256_aesenc_epi128(y2, y9); + y3 = _mm256_aesenc_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_encrypt_update_vaes_aes_enc_128_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y9); + y1 = _mm256_aesenclast_epi128(y1, y9); + y2 = _mm256_aesenclast_epi128(y2, y9); + y3 = _mm256_aesenclast_epi128(y3, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y9 = _mm256_srli_epi64(y4, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y4 = _mm256_slli_epi64(y4, 8); + y4 = _mm256_xor_si256(y4, y10); + y4 = _mm256_xor_si256(y4, y9); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y9 = _mm256_srli_epi64(y5, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y5, 8); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + y2 = _mm256_xor_si256(y2, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y9 = _mm256_srli_epi64(y6, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_slli_epi64(y6, 8); + y6 = _mm256_xor_si256(y6, y10); + y6 = _mm256_xor_si256(y6, y9); + y3 = _mm256_xor_si256(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + y9 = _mm256_srli_epi64(y7, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_slli_epi64(y7, 8); + y7 = _mm256_xor_si256(y7, y10); + y7 = _mm256_xor_si256(y7, y9); + r12 = (word32)((word32)r12 + 0x80); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_encrypt_update_vaes_enc_128; + } + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 0)); +L_AES_XTS_encrypt_update_vaes_done_128: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffffc0); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_encrypt_update_vaes_done_64; + } + /* 64 bytes of input */ + /* aes_enc_64 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + y9 = _mm256_srli_epi64(y4, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y4, 2); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + /* aes_enc_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y5); + y1 = _mm256_xor_si256(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + zf7 = (word32)r9; + zf8 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_encrypt_update_vaes_aes_enc_64_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + zf9 = (word32)r9; + zf10 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_encrypt_update_vaes_aes_enc_64_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_aesenc_epi128(y0, y9); + y1 = _mm256_aesenc_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_encrypt_update_vaes_aes_enc_64_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y9); + y1 = _mm256_aesenclast_epi128(y1, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y9 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y9), 31)); + y9 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y9), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + r12 = (word32)((word32)r12 + 0x40); +L_AES_XTS_encrypt_update_vaes_done_64: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffffe0); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_encrypt_update_vaes_done_32; + } + /* 32 bytes of input */ + /* aes_enc_32 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + /* aes_enc_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_aesenc_epi128(y0, y9); + zf11 = (word32)r9; + zf12 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_encrypt_update_vaes_aes_enc_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_aesenc_epi128(y0, y9); + zf13 = (word32)r9; + zf14 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_encrypt_update_vaes_aes_enc_32_aes_enc_block_last; + } + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_aesenc_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_encrypt_update_vaes_aes_enc_32_aes_enc_block_last: + y0 = _mm256_aesenclast_epi128(y0, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y9 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y9), 31)); + y9 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y9), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + r12 = (word32)((word32)r12 + 0x20); +L_AES_XTS_encrypt_update_vaes_done_32: + zf15 = (word32)r12; + zf16 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf15) == (zf16)) { + goto L_AES_XTS_encrypt_update_vaes_done_enc; + } + r11 = (word32)((word32)r11 - (word32)r12); + zf17 = (word32)r11; + zf18 = 0x10; + r11 = (word32)((word32)rax); + if ((zf17) < (zf18)) { + goto L_AES_XTS_encrypt_update_vaes_last_15; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + /* 16 bytes of input */ +L_AES_XTS_encrypt_update_vaes_enc_16: + rcx = (word64)(rdi + r12); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + /* aes_enc_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf19 = (word32)r9; + zf20 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_encrypt_update_vaes_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf21 = (word32)r9; + zf22 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_XTS_encrypt_update_vaes_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_encrypt_update_vaes_aes_enc_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y0)); + y4 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y4), 31)); + y4 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y4), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y4))); + r12 = (word32)((word32)r12 + 0x10); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_encrypt_update_vaes_enc_16; + } + if (((word32)r12) == ((word32)rax)) { + goto L_AES_XTS_encrypt_update_vaes_done_enc; + } +L_AES_XTS_encrypt_update_vaes_last_15: + r12 = (word64)(r12 - 0x10); + rcx = (word64)(rsi + r12); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + r12 = (word64)(r12 + 0x10); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(0); +L_AES_XTS_encrypt_update_vaes_last_15_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r12)) & 0xff); + WC_S8(rsi, r12) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r12 = (word32)((word32)r12 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r12) < ((word32)rax)) { + goto L_AES_XTS_encrypt_update_vaes_last_15_byte_loop; + } + r12 = (word64)(r12 - rdx); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + r12 = (word64)(r12 - 0x10); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + /* aes_enc_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf23 = (word32)r9; + zf24 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_XTS_encrypt_update_vaes_last_15_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf25 = (word32)r9; + zf26 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_XTS_encrypt_update_vaes_last_15_aes_enc_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_encrypt_update_vaes_last_15_aes_enc_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y0)); +L_AES_XTS_encrypt_update_vaes_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm256_castsi256_si128(y8)); +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_XTS_decrypt_vaes(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr) +{ + word64 rdi, rsi, rax, r12, r8, r9, r10, rsp, r13 = 0, r11 = 0, rcx = 0, + rdx = 0; + __m128i x12; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5, y6 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(), y8, y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y13, y14, y15; + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r12 = (word64)(size_t)i; + r8 = (word64)(size_t)key; + r9 = (word64)(size_t)key2; + r10 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_xts_gc_xts, 0)); + y13 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_xts_poly, 0))); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(L_vaes_aes_xts_shl, 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(L_vaes_aes_xts_shr, 0)); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + /* aes_enc_block */ + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 16))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 32))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 48))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 64))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 80))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, 96))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 112))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 128))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 144))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + zf1 = (word32)r10; + zf2 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_decrypt_vaes_tweak_aes_enc_block_last; + } + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 176))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y6))); + zf3 = (word32)r10; + zf4 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_decrypt_vaes_tweak_aes_enc_block_last; + } + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 208))); + y8 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 224))); +L_AES_XTS_decrypt_vaes_tweak_aes_enc_block_last: + y8 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y5))); + r13 = (word32)(0); + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_vaes_mul16_128; + } + r11 = (word32)((word32)r11 - 0x10); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_vaes_last_31_start; + } +L_AES_XTS_decrypt_vaes_mul16_128: + if (((word32)r11) < (0x20)) { + goto L_AES_XTS_decrypt_vaes_done_128; + } + if (((word32)r11) < (0x80)) { + goto L_AES_XTS_decrypt_vaes_done_128; + } + r11 = (word32)((word32)r11 & 0xffffff80); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + y9 = _mm256_srli_epi64(y4, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y4, 2); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + y9 = _mm256_srli_epi64(y5, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_slli_epi64(y5, 2); + y6 = _mm256_xor_si256(y6, y10); + y6 = _mm256_xor_si256(y6, y9); + y9 = _mm256_srli_epi64(y6, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_slli_epi64(y6, 2); + y7 = _mm256_xor_si256(y7, y10); + y7 = _mm256_xor_si256(y7, y9); +L_AES_XTS_decrypt_vaes_dec_128: + /* 128 bytes of input */ + /* aes_dec_128 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + /* aes_dec_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y5); + y1 = _mm256_xor_si256(y1, y9); + y2 = _mm256_xor_si256(y2, y6); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y7); + y3 = _mm256_xor_si256(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + zf5 = (word32)r10; + zf6 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_XTS_decrypt_vaes_aes_dec_128_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + zf7 = (word32)r10; + zf8 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_decrypt_vaes_aes_dec_128_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_vaes_aes_dec_128_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y9); + y1 = _mm256_aesdeclast_epi128(y1, y9); + y2 = _mm256_aesdeclast_epi128(y2, y9); + y3 = _mm256_aesdeclast_epi128(y3, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y9 = _mm256_srli_epi64(y4, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y4 = _mm256_slli_epi64(y4, 8); + y4 = _mm256_xor_si256(y4, y10); + y4 = _mm256_xor_si256(y4, y9); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y9 = _mm256_srli_epi64(y5, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y5, 8); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + y2 = _mm256_xor_si256(y2, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y9 = _mm256_srli_epi64(y6, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_slli_epi64(y6, 8); + y6 = _mm256_xor_si256(y6, y10); + y6 = _mm256_xor_si256(y6, y9); + y3 = _mm256_xor_si256(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + y9 = _mm256_srli_epi64(y7, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_slli_epi64(y7, 8); + y7 = _mm256_xor_si256(y7, y10); + y7 = _mm256_xor_si256(y7, y9); + r13 = (word32)((word32)r13 + 0x80); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_decrypt_vaes_dec_128; + } + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 0)); +L_AES_XTS_decrypt_vaes_done_128: + zf9 = (word32)r13; + zf10 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf9) == (zf10)) { + goto L_AES_XTS_decrypt_vaes_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_vaes_mul16_64; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r13); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_vaes_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r13); +L_AES_XTS_decrypt_vaes_mul16_64: + r11 = (word32)((word32)r11 & 0xffffffc0); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_decrypt_vaes_done_64; + } + /* 64 bytes of input */ + /* aes_dec_64 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + y9 = _mm256_srli_epi64(y4, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y4, 2); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + /* aes_dec_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y5); + y1 = _mm256_xor_si256(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + zf11 = (word32)r10; + zf12 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_decrypt_vaes_aes_dec_64_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + zf13 = (word32)r10; + zf14 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_decrypt_vaes_aes_dec_64_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_vaes_aes_dec_64_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y9); + y1 = _mm256_aesdeclast_epi128(y1, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y9 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y9), 31)); + y9 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y9), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + r13 = (word32)((word32)r13 + 0x40); +L_AES_XTS_decrypt_vaes_done_64: + zf15 = (word32)r13; + zf16 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf15) == (zf16)) { + goto L_AES_XTS_decrypt_vaes_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_vaes_mul16_32; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r13); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_vaes_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r13); +L_AES_XTS_decrypt_vaes_mul16_32: + r11 = (word32)((word32)r11 & 0xffffffe0); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_decrypt_vaes_done_32; + } + /* 32 bytes of input */ + /* aes_dec_32 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + /* aes_dec_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_aesdec_epi128(y0, y9); + zf17 = (word32)r10; + zf18 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_decrypt_vaes_aes_dec_32_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_aesdec_epi128(y0, y9); + zf19 = (word32)r10; + zf20 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_decrypt_vaes_aes_dec_32_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_vaes_aes_dec_32_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y9 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y9), 31)); + y9 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y9), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + r13 = (word32)((word32)r13 + 0x20); +L_AES_XTS_decrypt_vaes_done_32: + zf21 = (word32)r13; + zf22 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf21) == (zf22)) { + goto L_AES_XTS_decrypt_vaes_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_vaes_mul16; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r13); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_vaes_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r13); +L_AES_XTS_decrypt_vaes_mul16: +L_AES_XTS_decrypt_vaes_dec_16: + /* 16 bytes of input */ + rcx = (word64)(rdi + r13); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + /* aes_dec_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf23 = (word32)r10; + zf24 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_XTS_decrypt_vaes_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf25 = (word32)r10; + zf26 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_XTS_decrypt_vaes_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_vaes_aes_dec_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesdeclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y0)); + y4 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y4), 31)); + y4 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y4), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y4))); + r13 = (word32)((word32)r13 + 0x10); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_decrypt_vaes_dec_16; + } + if (((word32)r13) == ((word32)rax)) { + goto L_AES_XTS_decrypt_vaes_done_dec; + } +L_AES_XTS_decrypt_vaes_last_31_start: + y4 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y7 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y4), 31)); + y4 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y4), x12)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y4))); + rcx = (word64)(rdi + r13); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y7))); + /* aes_dec_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf27 = (word32)r10; + zf28 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_XTS_decrypt_vaes_last_31_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf29 = (word32)r10; + zf30 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_XTS_decrypt_vaes_last_31_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_vaes_last_31_aes_dec_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesdeclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y7))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); + r13 = (word64)(r13 + 0x10); + rdx = (word64)(0); +L_AES_XTS_decrypt_vaes_last_31_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r13)) & 0xff); + WC_S8(rsi, r13) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r13 = (word32)((word32)r13 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r13) < ((word32)rax)) { + goto L_AES_XTS_decrypt_vaes_last_31_byte_loop; + } + r13 = (word64)(r13 - rdx); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + /* aes_dec_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf31 = (word32)r10; + zf32 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_XTS_decrypt_vaes_last_31_2_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf33 = (word32)r10; + zf34 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_XTS_decrypt_vaes_last_31_2_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_vaes_last_31_2_aes_dec_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesdeclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + r13 = (word64)(r13 - 0x10); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y0)); +L_AES_XTS_decrypt_vaes_done_dec: + ; +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx2") +WOLFSSL_LOCAL void AES_XTS_decrypt_update_vaes(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr) +{ + word64 rdi, rsi, rax, r10, r8, r9, rsp, r12, r11, rcx = 0, rdx = 0; + __m128i x12; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), y8, + y9 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(), y13, + y14, y15; + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r10 = (word64)(size_t)key; + r8 = (word64)(size_t)tweak_block; + r9 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_vaes_aes_xts_gc_xts, 0)); + y13 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR( + L_vaes_aes_xts_poly, 0))); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(L_vaes_aes_xts_shl, 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(L_vaes_aes_xts_shr, 0)); + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + r12 = (word32)(0); + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_vaes_mul16_128; + } + r11 = (word32)((word32)r11 - 0x10); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_vaes_last_31_start; + } +L_AES_XTS_decrypt_update_vaes_mul16_128: + if (((word32)r11) < (0x20)) { + goto L_AES_XTS_decrypt_update_vaes_done_128; + } + if (((word32)r11) < (0x80)) { + goto L_AES_XTS_decrypt_update_vaes_done_128; + } + r11 = (word32)((word32)r11 & 0xffffff80); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + y9 = _mm256_srli_epi64(y4, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y4, 2); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + y9 = _mm256_srli_epi64(y5, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_slli_epi64(y5, 2); + y6 = _mm256_xor_si256(y6, y10); + y6 = _mm256_xor_si256(y6, y9); + y9 = _mm256_srli_epi64(y6, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_slli_epi64(y6, 2); + y7 = _mm256_xor_si256(y7, y10); + y7 = _mm256_xor_si256(y7, y9); +L_AES_XTS_decrypt_update_vaes_dec_128: + /* 128 bytes of input */ + /* aes_dec_128 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + /* aes_dec_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y5); + y1 = _mm256_xor_si256(y1, y9); + y2 = _mm256_xor_si256(y2, y6); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y7); + y3 = _mm256_xor_si256(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + zf1 = (word32)r9; + zf2 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_decrypt_update_vaes_aes_dec_128_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + zf3 = (word32)r9; + zf4 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_decrypt_update_vaes_aes_dec_128_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y2 = _mm256_aesdec_epi128(y2, y9); + y3 = _mm256_aesdec_epi128(y3, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_vaes_aes_dec_128_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y9); + y1 = _mm256_aesdeclast_epi128(y1, y9); + y2 = _mm256_aesdeclast_epi128(y2, y9); + y3 = _mm256_aesdeclast_epi128(y3, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y9 = _mm256_srli_epi64(y4, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y4 = _mm256_slli_epi64(y4, 8); + y4 = _mm256_xor_si256(y4, y10); + y4 = _mm256_xor_si256(y4, y9); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y9 = _mm256_srli_epi64(y5, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y5, 8); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + y2 = _mm256_xor_si256(y2, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + y9 = _mm256_srli_epi64(y6, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y6 = _mm256_slli_epi64(y6, 8); + y6 = _mm256_xor_si256(y6, y10); + y6 = _mm256_xor_si256(y6, y9); + y3 = _mm256_xor_si256(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + y9 = _mm256_srli_epi64(y7, 56); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y7 = _mm256_slli_epi64(y7, 8); + y7 = _mm256_xor_si256(y7, y10); + y7 = _mm256_xor_si256(y7, y9); + r12 = (word32)((word32)r12 + 0x80); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_decrypt_update_vaes_dec_128; + } + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 0)); +L_AES_XTS_decrypt_update_vaes_done_128: + zf5 = (word32)r12; + zf6 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf5) == (zf6)) { + goto L_AES_XTS_decrypt_update_vaes_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_vaes_mul16_64; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r12); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_vaes_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r12); +L_AES_XTS_decrypt_update_vaes_mul16_64: + r11 = (word32)((word32)r11 & 0xffffffc0); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_decrypt_update_vaes_done_64; + } + /* 64 bytes of input */ + /* aes_dec_64 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + y9 = _mm256_srli_epi64(y4, 62); + y10 = _mm256_clmulepi64_epi128(y9, y13, 1); + y9 = _mm256_bslli_epi128(y9, 8); + y5 = _mm256_slli_epi64(y4, 2); + y5 = _mm256_xor_si256(y5, y10); + y5 = _mm256_xor_si256(y5, y9); + /* aes_dec_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y1 = _mm256_xor_si256(y1, y5); + y1 = _mm256_xor_si256(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + zf7 = (word32)r9; + zf8 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_decrypt_update_vaes_aes_dec_64_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + zf9 = (word32)r9; + zf10 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_decrypt_update_vaes_aes_dec_64_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_aesdec_epi128(y0, y9); + y1 = _mm256_aesdec_epi128(y1, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_vaes_aes_dec_64_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y9); + y1 = _mm256_aesdeclast_epi128(y1, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y1 = _mm256_xor_si256(y1, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y9 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y9), 31)); + y9 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y9), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + r12 = (word32)((word32)r12 + 0x40); +L_AES_XTS_decrypt_update_vaes_done_64: + zf11 = (word32)r12; + zf12 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf11) == (zf12)) { + goto L_AES_XTS_decrypt_update_vaes_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_vaes_mul16_32; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r12); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_vaes_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r12); +L_AES_XTS_decrypt_update_vaes_mul16_32: + r11 = (word32)((word32)r11 & 0xffffffe0); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_decrypt_update_vaes_done_32; + } + /* 32 bytes of input */ + /* aes_dec_32 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y5 = _mm256_permute2x128_si256(y8, y8, 0); + y6 = _mm256_srlv_epi64(y5, y15); + y7 = _mm256_clmulepi64_epi128(y6, y13, 1); + y6 = _mm256_bslli_epi128(y6, 8); + y4 = _mm256_sllv_epi64(y5, y14); + y4 = _mm256_xor_si256(y4, y7); + y4 = _mm256_xor_si256(y4, y6); + /* aes_dec_block */ + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 0))); + y0 = _mm256_xor_si256(y0, y4); + y0 = _mm256_xor_si256(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_aesdec_epi128(y0, y9); + zf13 = (word32)r9; + zf14 = 0xb; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_decrypt_update_vaes_aes_dec_32_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_aesdec_epi128(y0, y9); + zf15 = (word32)r9; + zf16 = 0xd; + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_decrypt_update_vaes_aes_dec_32_aes_dec_block_last; + } + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_aesdec_epi128(y0, y9); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_vaes_aes_dec_32_aes_dec_block_last: + y0 = _mm256_aesdeclast_epi128(y0, y9); + y0 = _mm256_xor_si256(y0, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y9 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y9 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y9), 31)); + y9 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y9), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9))); + r12 = (word32)((word32)r12 + 0x20); +L_AES_XTS_decrypt_update_vaes_done_32: + zf17 = (word32)r12; + zf18 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf17) == (zf18)) { + goto L_AES_XTS_decrypt_update_vaes_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_vaes_mul16; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r12); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_vaes_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r12); +L_AES_XTS_decrypt_update_vaes_mul16: +L_AES_XTS_decrypt_update_vaes_dec_16: + /* 16 bytes of input */ + rcx = (word64)(rdi + r12); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + /* aes_dec_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf19 = (word32)r9; + zf20 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_decrypt_update_vaes_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf21 = (word32)r9; + zf22 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_XTS_decrypt_update_vaes_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_vaes_aes_dec_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesdeclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y0)); + y4 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y8 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y4), 31)); + y4 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y4), x12)); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y4))); + r12 = (word32)((word32)r12 + 0x10); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_decrypt_update_vaes_dec_16; + } + if (((word32)r12) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_vaes_done_dec; + } +L_AES_XTS_decrypt_update_vaes_last_31_start: + y4 = _mm256_zextsi128_si256(_mm_shuffle_epi32(_mm256_castsi256_si128(y8), + 0x13)); + y7 = _mm256_zextsi128_si256(_mm_add_epi64(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y8))); + y4 = _mm256_zextsi128_si256(_mm_srai_epi32(_mm256_castsi256_si128(y4), 31)); + y4 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y4), x12)); + y7 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y4))); + rcx = (word64)(rdi + r12); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y7))); + /* aes_dec_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf23 = (word32)r9; + zf24 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_XTS_decrypt_update_vaes_last_31_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf25 = (word32)r9; + zf26 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_XTS_decrypt_update_vaes_last_31_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_vaes_last_31_aes_dec_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesdeclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y7))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); + r12 = (word64)(r12 + 0x10); + rdx = (word64)(0); +L_AES_XTS_decrypt_update_vaes_last_31_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r12)) & 0xff); + WC_S8(rsi, r12) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r12 = (word32)((word32)r12 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r12) < ((word32)rax)) { + goto L_AES_XTS_decrypt_update_vaes_last_31_byte_loop; + } + r12 = (word64)(r12 - rdx); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + /* aes_dec_block */ + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + zf27 = (word32)r9; + zf28 = 0xb; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_XTS_decrypt_update_vaes_last_31_2_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + zf29 = (word32)r9; + zf30 = 0xd; + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_XTS_decrypt_update_vaes_last_31_2_aes_dec_block_last; + } + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + y0 = _mm256_zextsi128_si256(_mm_aesdec_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y6))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_vaes_last_31_2_aes_dec_block_last: + y0 = _mm256_zextsi128_si256(_mm_aesdeclast_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y5))); + y0 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + r12 = (word64)(r12 - 0x10); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y0)); +L_AES_XTS_decrypt_update_vaes_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm256_castsi256_si128(y8)); +} + +#endif /* HAVE_INTEL_VAES */ +#ifdef HAVE_INTEL_AVX512 +WC_X64I_TARGET("aes,avx") +WOLFSSL_LOCAL void AES_XTS_init_avx512(const unsigned char* i, + const unsigned char* key, int nr) +{ + word64 rdi, rsi, rdx; + __m128i x0, x2, x3 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + + rdi = (word64)(size_t)i; + rsi = (word64)(size_t)key; + rdx = (word64)(word32)nr; + + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + /* aes_enc_block */ + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x0 = _mm_aesenc_si128(x0, x2); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x0 = _mm_aesenc_si128(x0, x2); + zf1 = (word32)rdx; + zf2 = 0xb; + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_init_avx512_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x2); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x0 = _mm_aesenc_si128(x0, x3); + zf3 = (word32)rdx; + zf4 = 0xd; + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_init_avx512_tweak_aes_enc_block_last; + } + x0 = _mm_aesenc_si128(x0, x2); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x0 = _mm_aesenc_si128(x0, x3); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); +L_AES_XTS_init_avx512_tweak_aes_enc_block_last: + x0 = _mm_aesenclast_si128(x0, x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); +} + +XALIGNED(16) static const word32 L_avx512_aes_xts_gc_xts[] WC_X64I_UNUSED = { + 0x00000087, 0x00000000, 0x00000001, 0x00000000, +}; + +XALIGNED(16) static const word32 L_avx512_aes_xts_poly[] WC_X64I_UNUSED = { + 0x00000087, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_avx512_aes_xts_shl[] WC_X64I_UNUSED = { + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000001, 0x00000000, 0x00000001, 0x00000000, + 0x00000002, 0x00000000, 0x00000002, 0x00000000, + 0x00000003, 0x00000000, 0x00000003, 0x00000000, +}; + +XALIGNED(16) static const word32 L_avx512_aes_xts_shr[] WC_X64I_UNUSED = { + 0x00000040, 0x00000000, 0x00000040, 0x00000000, + 0x0000003f, 0x00000000, 0x0000003f, 0x00000000, + 0x0000003e, 0x00000000, 0x0000003e, 0x00000000, + 0x0000003d, 0x00000000, 0x0000003d, 0x00000000, +}; + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_XTS_encrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr) +{ + word64 rdi, rsi, rax, r12, r8, r9, r10, rsp, r13 = 0, r11 = 0, rcx = 0, + rdx = 0; + __m128i x12; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5, z6 = _mm512_setzero_si512(), + z7 = _mm512_setzero_si512(), z8, z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z13, z14, z15, + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(), + z22 = _mm512_setzero_si512(), z23 = _mm512_setzero_si512(), + z24 = _mm512_setzero_si512(), z25 = _mm512_setzero_si512(), + z26 = _mm512_setzero_si512(), z27 = _mm512_setzero_si512(), + z28 = _mm512_setzero_si512(), z29 = _mm512_setzero_si512(), + z30 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r12 = (word64)(size_t)i; + r8 = (word64)(size_t)key; + r9 = (word64)(size_t)key2; + r10 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_xts_gc_xts, 0)); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_xts_poly, 0))); + z14 = _mm512_loadu_si512((const void*)WC_PR(L_avx512_aes_xts_shl, 0)); + z15 = _mm512_loadu_si512((const void*)WC_PR(L_avx512_aes_xts_shr, 0)); + z8 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + /* aes_enc_block */ + z8 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z8), + _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 16))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 32))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 48))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 64))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 80))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 96))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 112))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 128))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 144))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + zf1 = (word32)r10; + zf2 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_encrypt_avx512_tweak_aes_enc_block_last; + } + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 176))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z6))); + zf3 = (word32)r10; + zf4 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_encrypt_avx512_tweak_aes_enc_block_last; + } + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 208))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 224))); +L_AES_XTS_encrypt_avx512_tweak_aes_enc_block_last: + z8 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + r13 = (word32)(0); + if (((word32)rax) < (0x20)) { + goto L_AES_XTS_encrypt_avx512_done_128; + } + z16 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + z17 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + z18 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + z19 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + z21 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + z23 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + z24 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + z25 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + z26 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)((word32)r10) < (sword32)(0xb)) { + goto L_AES_XTS_encrypt_avx512_key_cached; + } + z27 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + z28 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)((word32)r10) < (sword32)(0xd)) { + goto L_AES_XTS_encrypt_avx512_key_cached; + } + z29 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_encrypt_avx512_key_cached: + zf5 = (word32)rax; + zf6 = 0x100; + r11 = (word32)((word32)rax); + if ((zf5) < (zf6)) { + goto L_AES_XTS_encrypt_avx512_done_256; + } + r11 = (word32)((word32)r11 & 0xffffff00); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + z9 = _mm512_srli_epi64(z4, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z4, 4); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + z9 = _mm512_srli_epi64(z5, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_slli_epi64(z5, 4); + z6 = _mm512_ternarylogic_epi64(z6, z10, z9, 0x96); + z9 = _mm512_srli_epi64(z6, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_slli_epi64(z6, 4); + z7 = _mm512_ternarylogic_epi64(z7, z10, z9, 0x96); +L_AES_XTS_encrypt_avx512_enc_256: + /* 256 bytes of input */ + /* aes_enc_256 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + /* aes_enc_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z1 = _mm512_ternarylogic_epi64(z1, z16, z5, 0x96); + z2 = _mm512_ternarylogic_epi64(z2, z16, z6, 0x96); + z3 = _mm512_ternarylogic_epi64(z3, z16, z7, 0x96); + z0 = _mm512_aesenc_epi128(z0, z17); + z1 = _mm512_aesenc_epi128(z1, z17); + z2 = _mm512_aesenc_epi128(z2, z17); + z3 = _mm512_aesenc_epi128(z3, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z1 = _mm512_aesenc_epi128(z1, z18); + z2 = _mm512_aesenc_epi128(z2, z18); + z3 = _mm512_aesenc_epi128(z3, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z1 = _mm512_aesenc_epi128(z1, z19); + z2 = _mm512_aesenc_epi128(z2, z19); + z3 = _mm512_aesenc_epi128(z3, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z1 = _mm512_aesenc_epi128(z1, z20); + z2 = _mm512_aesenc_epi128(z2, z20); + z3 = _mm512_aesenc_epi128(z3, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z1 = _mm512_aesenc_epi128(z1, z21); + z2 = _mm512_aesenc_epi128(z2, z21); + z3 = _mm512_aesenc_epi128(z3, z21); + z0 = _mm512_aesenc_epi128(z0, z22); + z1 = _mm512_aesenc_epi128(z1, z22); + z2 = _mm512_aesenc_epi128(z2, z22); + z3 = _mm512_aesenc_epi128(z3, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z1 = _mm512_aesenc_epi128(z1, z23); + z2 = _mm512_aesenc_epi128(z2, z23); + z3 = _mm512_aesenc_epi128(z3, z23); + z0 = _mm512_aesenc_epi128(z0, z24); + z1 = _mm512_aesenc_epi128(z1, z24); + z2 = _mm512_aesenc_epi128(z2, z24); + z3 = _mm512_aesenc_epi128(z3, z24); + z0 = _mm512_aesenc_epi128(z0, z25); + z1 = _mm512_aesenc_epi128(z1, z25); + z2 = _mm512_aesenc_epi128(z2, z25); + z3 = _mm512_aesenc_epi128(z3, z25); + zf7 = (word32)r10; + zf8 = 0xb; + z9 = z26; + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z26); + z1 = _mm512_aesenc_epi128(z1, z26); + z2 = _mm512_aesenc_epi128(z2, z26); + z3 = _mm512_aesenc_epi128(z3, z26); + z0 = _mm512_aesenc_epi128(z0, z27); + z1 = _mm512_aesenc_epi128(z1, z27); + z2 = _mm512_aesenc_epi128(z2, z27); + z3 = _mm512_aesenc_epi128(z3, z27); + zf9 = (word32)r10; + zf10 = 0xd; + z9 = z28; + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z28); + z1 = _mm512_aesenc_epi128(z1, z28); + z2 = _mm512_aesenc_epi128(z2, z28); + z3 = _mm512_aesenc_epi128(z3, z28); + z0 = _mm512_aesenc_epi128(z0, z29); + z1 = _mm512_aesenc_epi128(z1, z29); + z2 = _mm512_aesenc_epi128(z2, z29); + z3 = _mm512_aesenc_epi128(z3, z29); + z9 = z30; +L_AES_XTS_encrypt_avx512_aes_enc_256_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z9); + z1 = _mm512_aesenclast_epi128(z1, z9); + z2 = _mm512_aesenclast_epi128(z2, z9); + z3 = _mm512_aesenclast_epi128(z3, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z9 = _mm512_srli_epi64(z4, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z4 = _mm512_slli_epi64(z4, 16); + z4 = _mm512_ternarylogic_epi64(z4, z10, z9, 0x96); + z1 = _mm512_xor_si512(z1, z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + z9 = _mm512_srli_epi64(z5, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z5, 16); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + z2 = _mm512_xor_si512(z2, z6); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z2); + z9 = _mm512_srli_epi64(z6, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_slli_epi64(z6, 16); + z6 = _mm512_ternarylogic_epi64(z6, z10, z9, 0x96); + z3 = _mm512_xor_si512(z3, z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z3); + z9 = _mm512_srli_epi64(z7, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_slli_epi64(z7, 16); + z7 = _mm512_ternarylogic_epi64(z7, z10, z9, 0x96); + r13 = (word32)((word32)r13 + 0x100); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_encrypt_avx512_enc_256; + } + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 0)); +L_AES_XTS_encrypt_avx512_done_256: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffff80); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_encrypt_avx512_done_128; + } + /* 128 bytes of input */ + /* aes_enc_128 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + z9 = _mm512_srli_epi64(z4, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z4, 4); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + /* aes_enc_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z1 = _mm512_ternarylogic_epi64(z1, z16, z5, 0x96); + z0 = _mm512_aesenc_epi128(z0, z17); + z1 = _mm512_aesenc_epi128(z1, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z1 = _mm512_aesenc_epi128(z1, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z1 = _mm512_aesenc_epi128(z1, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z1 = _mm512_aesenc_epi128(z1, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z1 = _mm512_aesenc_epi128(z1, z21); + z0 = _mm512_aesenc_epi128(z0, z22); + z1 = _mm512_aesenc_epi128(z1, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z1 = _mm512_aesenc_epi128(z1, z23); + z0 = _mm512_aesenc_epi128(z0, z24); + z1 = _mm512_aesenc_epi128(z1, z24); + z0 = _mm512_aesenc_epi128(z0, z25); + z1 = _mm512_aesenc_epi128(z1, z25); + zf11 = (word32)r10; + zf12 = 0xb; + z9 = z26; + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_128_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z26); + z1 = _mm512_aesenc_epi128(z1, z26); + z0 = _mm512_aesenc_epi128(z0, z27); + z1 = _mm512_aesenc_epi128(z1, z27); + zf13 = (word32)r10; + zf14 = 0xd; + z9 = z28; + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_128_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z28); + z1 = _mm512_aesenc_epi128(z1, z28); + z0 = _mm512_aesenc_epi128(z0, z29); + z1 = _mm512_aesenc_epi128(z1, z29); + z9 = z30; +L_AES_XTS_encrypt_avx512_aes_enc_128_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z9); + z1 = _mm512_aesenclast_epi128(z1, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z1 = _mm512_xor_si512(z1, z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 3)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z9 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z9), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z9), x12, 0x78)); + r13 = (word32)((word32)r13 + 0x80); +L_AES_XTS_encrypt_avx512_done_128: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffffc0); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_encrypt_avx512_done_64; + } + /* 64 bytes of input */ + /* aes_enc_64 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + /* aes_enc_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z0 = _mm512_aesenc_epi128(z0, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z0 = _mm512_aesenc_epi128(z0, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z0 = _mm512_aesenc_epi128(z0, z24); + z0 = _mm512_aesenc_epi128(z0, z25); + zf15 = (word32)r10; + zf16 = 0xb; + z9 = z26; + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z26); + z0 = _mm512_aesenc_epi128(z0, z27); + zf17 = (word32)r10; + zf18 = 0xd; + z9 = z28; + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z28); + z0 = _mm512_aesenc_epi128(z0, z29); + z9 = z30; +L_AES_XTS_encrypt_avx512_aes_enc_64_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 3)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z9 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z9), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z9), x12, 0x78)); + r13 = (word32)((word32)r13 + 0x40); +L_AES_XTS_encrypt_avx512_done_64: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffffe0); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_encrypt_avx512_done_32; + } + /* 32 bytes of input */ + /* aes_enc_32 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + /* aes_enc_block */ + z0 = _mm512_zextsi256_si512(_mm256_ternarylogic_epi64( + _mm512_castsi512_si256(z0), _mm512_castsi512_si256(z16), + _mm512_castsi512_si256(z4), 0x96)); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z17))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z18))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z19))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z20))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z21))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z22))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z23))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z24))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z25))); + zf19 = (word32)r10; + zf20 = 0xb; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z26)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_32_aes_enc_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z26))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z27))); + zf21 = (word32)r10; + zf22 = 0xd; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z28)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_32_aes_enc_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z28))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z29))); + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z30)); +L_AES_XTS_encrypt_avx512_aes_enc_32_aes_enc_block_last: + z0 = _mm512_zextsi256_si512(_mm256_aesenclast_epi128(_mm512_castsi512_si256( + z0), _mm512_castsi512_si256(z9))); + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z4))); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + r13 = (word32)((word32)r13 + 0x20); +L_AES_XTS_encrypt_avx512_done_32: + zf23 = (word32)r13; + zf24 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf23) == (zf24)) { + goto L_AES_XTS_encrypt_avx512_done_enc; + } + r11 = (word32)((word32)r11 - (word32)r13); + zf25 = (word32)r11; + zf26 = 0x10; + r11 = (word32)((word32)rax); + if ((zf25) < (zf26)) { + goto L_AES_XTS_encrypt_avx512_last_15; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + /* 16 bytes of input */ +L_AES_XTS_encrypt_avx512_enc_16: + rcx = (word64)(rdi + r13); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + /* aes_enc_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf27 = (word32)r10; + zf28 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf29 = (word32)r10; + zf30 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_XTS_encrypt_avx512_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_encrypt_avx512_aes_enc_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + z4 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z4 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z4), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z4), x12, 0x78)); + r13 = (word32)((word32)r13 + 0x10); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_encrypt_avx512_enc_16; + } + if (((word32)r13) == ((word32)rax)) { + goto L_AES_XTS_encrypt_avx512_done_enc; + } +L_AES_XTS_encrypt_avx512_last_15: + r13 = (word64)(r13 - 0x10); + rcx = (word64)(rsi + r13); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + r13 = (word64)(r13 + 0x10); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(0); +L_AES_XTS_encrypt_avx512_last_15_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r13)) & 0xff); + WC_S8(rsi, r13) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r13 = (word32)((word32)r13 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r13) < ((word32)rax)) { + goto L_AES_XTS_encrypt_avx512_last_15_byte_loop; + } + r13 = (word64)(r13 - rdx); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + r13 = (word64)(r13 - 0x10); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + /* aes_enc_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf31 = (word32)r10; + zf32 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_XTS_encrypt_avx512_last_15_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf33 = (word32)r10; + zf34 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_XTS_encrypt_avx512_last_15_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_encrypt_avx512_last_15_aes_enc_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); +L_AES_XTS_encrypt_avx512_done_enc: + ; +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_XTS_encrypt_update_avx512(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr) +{ + word64 rdi, rsi, rax, r10, r8, r9, rsp, r12, r11 = 0, rcx = 0, rdx = 0; + __m128i x12; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), z8, + z9 = _mm512_setzero_si512(), z10 = _mm512_setzero_si512(), z13, + z14, z15, z16 = _mm512_setzero_si512(), + z17 = _mm512_setzero_si512(), z18 = _mm512_setzero_si512(), + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(), + z23 = _mm512_setzero_si512(), z24 = _mm512_setzero_si512(), + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(), + z27 = _mm512_setzero_si512(), z28 = _mm512_setzero_si512(), + z29 = _mm512_setzero_si512(), z30 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r10 = (word64)(size_t)key; + r8 = (word64)(size_t)tweak_block; + r9 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_xts_gc_xts, 0)); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_xts_poly, 0))); + z14 = _mm512_loadu_si512((const void*)WC_PR(L_avx512_aes_xts_shl, 0)); + z15 = _mm512_loadu_si512((const void*)WC_PR(L_avx512_aes_xts_shr, 0)); + z8 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + r12 = (word32)(0); + if (((word32)rax) < (0x20)) { + goto L_AES_XTS_encrypt_update_avx512_done_128; + } + z16 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 0))); + z17 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + z18 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + z19 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + z21 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + z23 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + z24 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + z25 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + z26 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)((word32)r9) < (sword32)(0xb)) { + goto L_AES_XTS_encrypt_update_avx512_key_cached; + } + z27 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + z28 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)((word32)r9) < (sword32)(0xd)) { + goto L_AES_XTS_encrypt_update_avx512_key_cached; + } + z29 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_encrypt_update_avx512_key_cached: + zf1 = (word32)rax; + zf2 = 0x100; + r11 = (word32)((word32)rax); + if ((zf1) < (zf2)) { + goto L_AES_XTS_encrypt_update_avx512_done_256; + } + r11 = (word32)((word32)r11 & 0xffffff00); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + z9 = _mm512_srli_epi64(z4, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z4, 4); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + z9 = _mm512_srli_epi64(z5, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_slli_epi64(z5, 4); + z6 = _mm512_ternarylogic_epi64(z6, z10, z9, 0x96); + z9 = _mm512_srli_epi64(z6, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_slli_epi64(z6, 4); + z7 = _mm512_ternarylogic_epi64(z7, z10, z9, 0x96); +L_AES_XTS_encrypt_update_avx512_enc_256: + /* 256 bytes of input */ + /* aes_enc_256 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + /* aes_enc_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z1 = _mm512_ternarylogic_epi64(z1, z16, z5, 0x96); + z2 = _mm512_ternarylogic_epi64(z2, z16, z6, 0x96); + z3 = _mm512_ternarylogic_epi64(z3, z16, z7, 0x96); + z0 = _mm512_aesenc_epi128(z0, z17); + z1 = _mm512_aesenc_epi128(z1, z17); + z2 = _mm512_aesenc_epi128(z2, z17); + z3 = _mm512_aesenc_epi128(z3, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z1 = _mm512_aesenc_epi128(z1, z18); + z2 = _mm512_aesenc_epi128(z2, z18); + z3 = _mm512_aesenc_epi128(z3, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z1 = _mm512_aesenc_epi128(z1, z19); + z2 = _mm512_aesenc_epi128(z2, z19); + z3 = _mm512_aesenc_epi128(z3, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z1 = _mm512_aesenc_epi128(z1, z20); + z2 = _mm512_aesenc_epi128(z2, z20); + z3 = _mm512_aesenc_epi128(z3, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z1 = _mm512_aesenc_epi128(z1, z21); + z2 = _mm512_aesenc_epi128(z2, z21); + z3 = _mm512_aesenc_epi128(z3, z21); + z0 = _mm512_aesenc_epi128(z0, z22); + z1 = _mm512_aesenc_epi128(z1, z22); + z2 = _mm512_aesenc_epi128(z2, z22); + z3 = _mm512_aesenc_epi128(z3, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z1 = _mm512_aesenc_epi128(z1, z23); + z2 = _mm512_aesenc_epi128(z2, z23); + z3 = _mm512_aesenc_epi128(z3, z23); + z0 = _mm512_aesenc_epi128(z0, z24); + z1 = _mm512_aesenc_epi128(z1, z24); + z2 = _mm512_aesenc_epi128(z2, z24); + z3 = _mm512_aesenc_epi128(z3, z24); + z0 = _mm512_aesenc_epi128(z0, z25); + z1 = _mm512_aesenc_epi128(z1, z25); + z2 = _mm512_aesenc_epi128(z2, z25); + z3 = _mm512_aesenc_epi128(z3, z25); + zf3 = (word32)r9; + zf4 = 0xb; + z9 = z26; + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z26); + z1 = _mm512_aesenc_epi128(z1, z26); + z2 = _mm512_aesenc_epi128(z2, z26); + z3 = _mm512_aesenc_epi128(z3, z26); + z0 = _mm512_aesenc_epi128(z0, z27); + z1 = _mm512_aesenc_epi128(z1, z27); + z2 = _mm512_aesenc_epi128(z2, z27); + z3 = _mm512_aesenc_epi128(z3, z27); + zf5 = (word32)r9; + zf6 = 0xd; + z9 = z28; + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_256_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z28); + z1 = _mm512_aesenc_epi128(z1, z28); + z2 = _mm512_aesenc_epi128(z2, z28); + z3 = _mm512_aesenc_epi128(z3, z28); + z0 = _mm512_aesenc_epi128(z0, z29); + z1 = _mm512_aesenc_epi128(z1, z29); + z2 = _mm512_aesenc_epi128(z2, z29); + z3 = _mm512_aesenc_epi128(z3, z29); + z9 = z30; +L_AES_XTS_encrypt_update_avx512_aes_enc_256_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z9); + z1 = _mm512_aesenclast_epi128(z1, z9); + z2 = _mm512_aesenclast_epi128(z2, z9); + z3 = _mm512_aesenclast_epi128(z3, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z9 = _mm512_srli_epi64(z4, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z4 = _mm512_slli_epi64(z4, 16); + z4 = _mm512_ternarylogic_epi64(z4, z10, z9, 0x96); + z1 = _mm512_xor_si512(z1, z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + z9 = _mm512_srli_epi64(z5, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z5, 16); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + z2 = _mm512_xor_si512(z2, z6); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z2); + z9 = _mm512_srli_epi64(z6, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_slli_epi64(z6, 16); + z6 = _mm512_ternarylogic_epi64(z6, z10, z9, 0x96); + z3 = _mm512_xor_si512(z3, z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z3); + z9 = _mm512_srli_epi64(z7, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_slli_epi64(z7, 16); + z7 = _mm512_ternarylogic_epi64(z7, z10, z9, 0x96); + r12 = (word32)((word32)r12 + 0x100); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_encrypt_update_avx512_enc_256; + } + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 0)); +L_AES_XTS_encrypt_update_avx512_done_256: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffff80); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_encrypt_update_avx512_done_128; + } + /* 128 bytes of input */ + /* aes_enc_128 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + z9 = _mm512_srli_epi64(z4, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z4, 4); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + /* aes_enc_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z1 = _mm512_ternarylogic_epi64(z1, z16, z5, 0x96); + z0 = _mm512_aesenc_epi128(z0, z17); + z1 = _mm512_aesenc_epi128(z1, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z1 = _mm512_aesenc_epi128(z1, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z1 = _mm512_aesenc_epi128(z1, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z1 = _mm512_aesenc_epi128(z1, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z1 = _mm512_aesenc_epi128(z1, z21); + z0 = _mm512_aesenc_epi128(z0, z22); + z1 = _mm512_aesenc_epi128(z1, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z1 = _mm512_aesenc_epi128(z1, z23); + z0 = _mm512_aesenc_epi128(z0, z24); + z1 = _mm512_aesenc_epi128(z1, z24); + z0 = _mm512_aesenc_epi128(z0, z25); + z1 = _mm512_aesenc_epi128(z1, z25); + zf7 = (word32)r9; + zf8 = 0xb; + z9 = z26; + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_128_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z26); + z1 = _mm512_aesenc_epi128(z1, z26); + z0 = _mm512_aesenc_epi128(z0, z27); + z1 = _mm512_aesenc_epi128(z1, z27); + zf9 = (word32)r9; + zf10 = 0xd; + z9 = z28; + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_128_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z28); + z1 = _mm512_aesenc_epi128(z1, z28); + z0 = _mm512_aesenc_epi128(z0, z29); + z1 = _mm512_aesenc_epi128(z1, z29); + z9 = z30; +L_AES_XTS_encrypt_update_avx512_aes_enc_128_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z9); + z1 = _mm512_aesenclast_epi128(z1, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z1 = _mm512_xor_si512(z1, z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 3)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z9 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z9), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z9), x12, 0x78)); + r12 = (word32)((word32)r12 + 0x80); +L_AES_XTS_encrypt_update_avx512_done_128: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffffc0); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_encrypt_update_avx512_done_64; + } + /* 64 bytes of input */ + /* aes_enc_64 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + /* aes_enc_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z0 = _mm512_aesenc_epi128(z0, z17); + z0 = _mm512_aesenc_epi128(z0, z18); + z0 = _mm512_aesenc_epi128(z0, z19); + z0 = _mm512_aesenc_epi128(z0, z20); + z0 = _mm512_aesenc_epi128(z0, z21); + z0 = _mm512_aesenc_epi128(z0, z22); + z0 = _mm512_aesenc_epi128(z0, z23); + z0 = _mm512_aesenc_epi128(z0, z24); + z0 = _mm512_aesenc_epi128(z0, z25); + zf11 = (word32)r9; + zf12 = 0xb; + z9 = z26; + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z26); + z0 = _mm512_aesenc_epi128(z0, z27); + zf13 = (word32)r9; + zf14 = 0xd; + z9 = z28; + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_64_aes_enc_block_last; + } + z0 = _mm512_aesenc_epi128(z0, z28); + z0 = _mm512_aesenc_epi128(z0, z29); + z9 = z30; +L_AES_XTS_encrypt_update_avx512_aes_enc_64_aes_enc_block_last: + z0 = _mm512_aesenclast_epi128(z0, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 3)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z9 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z9), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z9), x12, 0x78)); + r12 = (word32)((word32)r12 + 0x40); +L_AES_XTS_encrypt_update_avx512_done_64: + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xffffffe0); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_encrypt_update_avx512_done_32; + } + /* 32 bytes of input */ + /* aes_enc_32 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + /* aes_enc_block */ + z0 = _mm512_zextsi256_si512(_mm256_ternarylogic_epi64( + _mm512_castsi512_si256(z0), _mm512_castsi512_si256(z16), + _mm512_castsi512_si256(z4), 0x96)); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z17))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z18))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z19))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z20))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z21))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z22))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z23))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z24))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z25))); + zf15 = (word32)r9; + zf16 = 0xb; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z26)); + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_32_aes_enc_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z26))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z27))); + zf17 = (word32)r9; + zf18 = 0xd; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z28)); + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_32_aes_enc_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z28))); + z0 = _mm512_zextsi256_si512(_mm256_aesenc_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z29))); + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z30)); +L_AES_XTS_encrypt_update_avx512_aes_enc_32_aes_enc_block_last: + z0 = _mm512_zextsi256_si512(_mm256_aesenclast_epi128(_mm512_castsi512_si256( + z0), _mm512_castsi512_si256(z9))); + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z4))); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + r12 = (word32)((word32)r12 + 0x20); +L_AES_XTS_encrypt_update_avx512_done_32: + zf19 = (word32)r12; + zf20 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf19) == (zf20)) { + goto L_AES_XTS_encrypt_update_avx512_done_enc; + } + r11 = (word32)((word32)r11 - (word32)r12); + zf21 = (word32)r11; + zf22 = 0x10; + r11 = (word32)((word32)rax); + if ((zf21) < (zf22)) { + goto L_AES_XTS_encrypt_update_avx512_last_15; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + /* 16 bytes of input */ +L_AES_XTS_encrypt_update_avx512_enc_16: + rcx = (word64)(rdi + r12); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + /* aes_enc_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf23 = (word32)r9; + zf24 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf25 = (word32)r9; + zf26 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_XTS_encrypt_update_avx512_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_encrypt_update_avx512_aes_enc_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + z4 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z4 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z4), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z4), x12, 0x78)); + r12 = (word32)((word32)r12 + 0x10); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_encrypt_update_avx512_enc_16; + } + if (((word32)r12) == ((word32)rax)) { + goto L_AES_XTS_encrypt_update_avx512_done_enc; + } +L_AES_XTS_encrypt_update_avx512_last_15: + r12 = (word64)(r12 - 0x10); + rcx = (word64)(rsi + r12); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + r12 = (word64)(r12 + 0x10); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(0); +L_AES_XTS_encrypt_update_avx512_last_15_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r12)) & 0xff); + WC_S8(rsi, r12) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r12 = (word32)((word32)r12 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r12) < ((word32)rax)) { + goto L_AES_XTS_encrypt_update_avx512_last_15_byte_loop; + } + r12 = (word64)(r12 - rdx); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + r12 = (word64)(r12 - 0x10); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + /* aes_enc_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf27 = (word32)r9; + zf28 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_XTS_encrypt_update_avx512_last_15_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf29 = (word32)r9; + zf30 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_XTS_encrypt_update_avx512_last_15_aes_enc_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_encrypt_update_avx512_last_15_aes_enc_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); +L_AES_XTS_encrypt_update_avx512_done_enc: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z8)); +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_XTS_decrypt_avx512(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* i, + const unsigned char* key, const unsigned char* key2, int nr) +{ + word64 rdi, rsi, rax, r12, r8, r9, r10, rsp, r13 = 0, r11 = 0, rcx = 0, + rdx = 0; + __m128i x12; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5, z6 = _mm512_setzero_si512(), + z7 = _mm512_setzero_si512(), z8, z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z13, z14, z15, + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(), + z22 = _mm512_setzero_si512(), z23 = _mm512_setzero_si512(), + z24 = _mm512_setzero_si512(), z25 = _mm512_setzero_si512(), + z26 = _mm512_setzero_si512(), z27 = _mm512_setzero_si512(), + z28 = _mm512_setzero_si512(), z29 = _mm512_setzero_si512(), + z30 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + word32 zf35; + word32 zf36; + word32 zf37; + word32 zf38; + word32 zf39; + word32 zf40; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r12 = (word64)(size_t)i; + r8 = (word64)(size_t)key; + r9 = (word64)(size_t)key2; + r10 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_xts_gc_xts, 0)); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_xts_poly, 0))); + z14 = _mm512_loadu_si512((const void*)WC_PR(L_avx512_aes_xts_shl, 0)); + z15 = _mm512_loadu_si512((const void*)WC_PR(L_avx512_aes_xts_shr, 0)); + z8 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + /* aes_enc_block */ + z8 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z8), + _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 16))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 32))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 48))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 64))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 80))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, 96))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 112))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 128))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 144))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + zf1 = (word32)r10; + zf2 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 160))); + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_decrypt_avx512_tweak_aes_enc_block_last; + } + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 176))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z6))); + zf3 = (word32)r10; + zf4 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 192))); + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_decrypt_avx512_tweak_aes_enc_block_last; + } + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 208))); + z8 = _mm512_zextsi128_si512(_mm_aesenc_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r9, + 224))); +L_AES_XTS_decrypt_avx512_tweak_aes_enc_block_last: + z8 = _mm512_zextsi128_si512(_mm_aesenclast_si128(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z5))); + r13 = (word32)(0); + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_avx512_mul16_256; + } + r11 = (word32)((word32)r11 - 0x10); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_avx512_last_31_start; + } +L_AES_XTS_decrypt_avx512_mul16_256: + if (((word32)r11) < (0x20)) { + goto L_AES_XTS_decrypt_avx512_done_128; + } + z16 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + z17 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 16))); + z18 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 32))); + z19 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 48))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 64))); + z21 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 80))); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 96))); + z23 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + z24 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + z25 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + z26 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)((word32)r10) < (sword32)(0xb)) { + goto L_AES_XTS_decrypt_avx512_key_cached; + } + z27 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + z28 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)((word32)r10) < (sword32)(0xd)) { + goto L_AES_XTS_decrypt_avx512_key_cached; + } + z29 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_avx512_key_cached: + if (((word32)r11) < (0x100)) { + goto L_AES_XTS_decrypt_avx512_done_256; + } + r11 = (word32)((word32)r11 & 0xffffff00); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + z9 = _mm512_srli_epi64(z4, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z4, 4); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + z9 = _mm512_srli_epi64(z5, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_slli_epi64(z5, 4); + z6 = _mm512_ternarylogic_epi64(z6, z10, z9, 0x96); + z9 = _mm512_srli_epi64(z6, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_slli_epi64(z6, 4); + z7 = _mm512_ternarylogic_epi64(z7, z10, z9, 0x96); +L_AES_XTS_decrypt_avx512_dec_256: + /* 256 bytes of input */ + /* aes_dec_256 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + /* aes_dec_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z1 = _mm512_ternarylogic_epi64(z1, z16, z5, 0x96); + z2 = _mm512_ternarylogic_epi64(z2, z16, z6, 0x96); + z3 = _mm512_ternarylogic_epi64(z3, z16, z7, 0x96); + z0 = _mm512_aesdec_epi128(z0, z17); + z1 = _mm512_aesdec_epi128(z1, z17); + z2 = _mm512_aesdec_epi128(z2, z17); + z3 = _mm512_aesdec_epi128(z3, z17); + z0 = _mm512_aesdec_epi128(z0, z18); + z1 = _mm512_aesdec_epi128(z1, z18); + z2 = _mm512_aesdec_epi128(z2, z18); + z3 = _mm512_aesdec_epi128(z3, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z1 = _mm512_aesdec_epi128(z1, z19); + z2 = _mm512_aesdec_epi128(z2, z19); + z3 = _mm512_aesdec_epi128(z3, z19); + z0 = _mm512_aesdec_epi128(z0, z20); + z1 = _mm512_aesdec_epi128(z1, z20); + z2 = _mm512_aesdec_epi128(z2, z20); + z3 = _mm512_aesdec_epi128(z3, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z1 = _mm512_aesdec_epi128(z1, z21); + z2 = _mm512_aesdec_epi128(z2, z21); + z3 = _mm512_aesdec_epi128(z3, z21); + z0 = _mm512_aesdec_epi128(z0, z22); + z1 = _mm512_aesdec_epi128(z1, z22); + z2 = _mm512_aesdec_epi128(z2, z22); + z3 = _mm512_aesdec_epi128(z3, z22); + z0 = _mm512_aesdec_epi128(z0, z23); + z1 = _mm512_aesdec_epi128(z1, z23); + z2 = _mm512_aesdec_epi128(z2, z23); + z3 = _mm512_aesdec_epi128(z3, z23); + z0 = _mm512_aesdec_epi128(z0, z24); + z1 = _mm512_aesdec_epi128(z1, z24); + z2 = _mm512_aesdec_epi128(z2, z24); + z3 = _mm512_aesdec_epi128(z3, z24); + z0 = _mm512_aesdec_epi128(z0, z25); + z1 = _mm512_aesdec_epi128(z1, z25); + z2 = _mm512_aesdec_epi128(z2, z25); + z3 = _mm512_aesdec_epi128(z3, z25); + zf5 = (word32)r10; + zf6 = 0xb; + z9 = z26; + if ((sword32)(zf5) < (sword32)(zf6)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_256_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z26); + z1 = _mm512_aesdec_epi128(z1, z26); + z2 = _mm512_aesdec_epi128(z2, z26); + z3 = _mm512_aesdec_epi128(z3, z26); + z0 = _mm512_aesdec_epi128(z0, z27); + z1 = _mm512_aesdec_epi128(z1, z27); + z2 = _mm512_aesdec_epi128(z2, z27); + z3 = _mm512_aesdec_epi128(z3, z27); + zf7 = (word32)r10; + zf8 = 0xd; + z9 = z28; + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_256_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z28); + z1 = _mm512_aesdec_epi128(z1, z28); + z2 = _mm512_aesdec_epi128(z2, z28); + z3 = _mm512_aesdec_epi128(z3, z28); + z0 = _mm512_aesdec_epi128(z0, z29); + z1 = _mm512_aesdec_epi128(z1, z29); + z2 = _mm512_aesdec_epi128(z2, z29); + z3 = _mm512_aesdec_epi128(z3, z29); + z9 = z30; +L_AES_XTS_decrypt_avx512_aes_dec_256_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z9); + z1 = _mm512_aesdeclast_epi128(z1, z9); + z2 = _mm512_aesdeclast_epi128(z2, z9); + z3 = _mm512_aesdeclast_epi128(z3, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z9 = _mm512_srli_epi64(z4, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z4 = _mm512_slli_epi64(z4, 16); + z4 = _mm512_ternarylogic_epi64(z4, z10, z9, 0x96); + z1 = _mm512_xor_si512(z1, z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + z9 = _mm512_srli_epi64(z5, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z5, 16); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + z2 = _mm512_xor_si512(z2, z6); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z2); + z9 = _mm512_srli_epi64(z6, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_slli_epi64(z6, 16); + z6 = _mm512_ternarylogic_epi64(z6, z10, z9, 0x96); + z3 = _mm512_xor_si512(z3, z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z3); + z9 = _mm512_srli_epi64(z7, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_slli_epi64(z7, 16); + z7 = _mm512_ternarylogic_epi64(z7, z10, z9, 0x96); + r13 = (word32)((word32)r13 + 0x100); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_decrypt_avx512_dec_256; + } + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 0)); +L_AES_XTS_decrypt_avx512_done_256: + zf9 = (word32)r13; + zf10 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf9) == (zf10)) { + goto L_AES_XTS_decrypt_avx512_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_avx512_mul16_128; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r13); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_avx512_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r13); +L_AES_XTS_decrypt_avx512_mul16_128: + r11 = (word32)((word32)r11 & 0xffffff80); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_decrypt_avx512_done_128; + } + /* 128 bytes of input */ + /* aes_dec_128 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + z9 = _mm512_srli_epi64(z4, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z4, 4); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + /* aes_dec_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z1 = _mm512_ternarylogic_epi64(z1, z16, z5, 0x96); + z0 = _mm512_aesdec_epi128(z0, z17); + z1 = _mm512_aesdec_epi128(z1, z17); + z0 = _mm512_aesdec_epi128(z0, z18); + z1 = _mm512_aesdec_epi128(z1, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z1 = _mm512_aesdec_epi128(z1, z19); + z0 = _mm512_aesdec_epi128(z0, z20); + z1 = _mm512_aesdec_epi128(z1, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z1 = _mm512_aesdec_epi128(z1, z21); + z0 = _mm512_aesdec_epi128(z0, z22); + z1 = _mm512_aesdec_epi128(z1, z22); + z0 = _mm512_aesdec_epi128(z0, z23); + z1 = _mm512_aesdec_epi128(z1, z23); + z0 = _mm512_aesdec_epi128(z0, z24); + z1 = _mm512_aesdec_epi128(z1, z24); + z0 = _mm512_aesdec_epi128(z0, z25); + z1 = _mm512_aesdec_epi128(z1, z25); + zf11 = (word32)r10; + zf12 = 0xb; + z9 = z26; + if ((sword32)(zf11) < (sword32)(zf12)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_128_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z26); + z1 = _mm512_aesdec_epi128(z1, z26); + z0 = _mm512_aesdec_epi128(z0, z27); + z1 = _mm512_aesdec_epi128(z1, z27); + zf13 = (word32)r10; + zf14 = 0xd; + z9 = z28; + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_128_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z28); + z1 = _mm512_aesdec_epi128(z1, z28); + z0 = _mm512_aesdec_epi128(z0, z29); + z1 = _mm512_aesdec_epi128(z1, z29); + z9 = z30; +L_AES_XTS_decrypt_avx512_aes_dec_128_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z9); + z1 = _mm512_aesdeclast_epi128(z1, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z1 = _mm512_xor_si512(z1, z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 3)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z9 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z9), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z9), x12, 0x78)); + r13 = (word32)((word32)r13 + 0x80); +L_AES_XTS_decrypt_avx512_done_128: + zf15 = (word32)r13; + zf16 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf15) == (zf16)) { + goto L_AES_XTS_decrypt_avx512_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_avx512_mul16_64; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r13); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_avx512_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r13); +L_AES_XTS_decrypt_avx512_mul16_64: + r11 = (word32)((word32)r11 & 0xffffffc0); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_decrypt_avx512_done_64; + } + /* 64 bytes of input */ + /* aes_dec_64 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + /* aes_dec_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z0 = _mm512_aesdec_epi128(z0, z17); + z0 = _mm512_aesdec_epi128(z0, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z0 = _mm512_aesdec_epi128(z0, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z0 = _mm512_aesdec_epi128(z0, z22); + z0 = _mm512_aesdec_epi128(z0, z23); + z0 = _mm512_aesdec_epi128(z0, z24); + z0 = _mm512_aesdec_epi128(z0, z25); + zf17 = (word32)r10; + zf18 = 0xb; + z9 = z26; + if ((sword32)(zf17) < (sword32)(zf18)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_64_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z26); + z0 = _mm512_aesdec_epi128(z0, z27); + zf19 = (word32)r10; + zf20 = 0xd; + z9 = z28; + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_64_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z28); + z0 = _mm512_aesdec_epi128(z0, z29); + z9 = z30; +L_AES_XTS_decrypt_avx512_aes_dec_64_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 3)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z9 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z9), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z9), x12, 0x78)); + r13 = (word32)((word32)r13 + 0x40); +L_AES_XTS_decrypt_avx512_done_64: + zf21 = (word32)r13; + zf22 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf21) == (zf22)) { + goto L_AES_XTS_decrypt_avx512_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_avx512_mul16_32; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r13); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_avx512_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r13); +L_AES_XTS_decrypt_avx512_mul16_32: + r11 = (word32)((word32)r11 & 0xffffffe0); + if (((word32)r13) == ((word32)r11)) { + goto L_AES_XTS_decrypt_avx512_done_32; + } + /* 32 bytes of input */ + /* aes_dec_32 */ + rcx = (word64)(rdi + r13); + rdx = (word64)(rsi + r13); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + /* aes_dec_block */ + z0 = _mm512_zextsi256_si512(_mm256_ternarylogic_epi64( + _mm512_castsi512_si256(z0), _mm512_castsi512_si256(z16), + _mm512_castsi512_si256(z4), 0x96)); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z17))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z18))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z19))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z20))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z21))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z22))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z23))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z24))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z25))); + zf23 = (word32)r10; + zf24 = 0xb; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z26)); + if ((sword32)(zf23) < (sword32)(zf24)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_32_aes_dec_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z26))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z27))); + zf25 = (word32)r10; + zf26 = 0xd; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z28)); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_32_aes_dec_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z28))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z29))); + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z30)); +L_AES_XTS_decrypt_avx512_aes_dec_32_aes_dec_block_last: + z0 = _mm512_zextsi256_si512(_mm256_aesdeclast_epi128(_mm512_castsi512_si256( + z0), _mm512_castsi512_si256(z9))); + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z4))); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + r13 = (word32)((word32)r13 + 0x20); +L_AES_XTS_decrypt_avx512_done_32: + zf27 = (word32)r13; + zf28 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf27) == (zf28)) { + goto L_AES_XTS_decrypt_avx512_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_avx512_mul16; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r13); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_avx512_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r13); +L_AES_XTS_decrypt_avx512_mul16: +L_AES_XTS_decrypt_avx512_dec_16: + /* 16 bytes of input */ + rcx = (word64)(rdi + r13); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + /* aes_dec_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf29 = (word32)r10; + zf30 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf31 = (word32)r10; + zf32 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_XTS_decrypt_avx512_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_avx512_aes_dec_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesdeclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + z4 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z4 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z4), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z4), x12, 0x78)); + r13 = (word32)((word32)r13 + 0x10); + if (((word32)r13) < ((word32)r11)) { + goto L_AES_XTS_decrypt_avx512_dec_16; + } + if (((word32)r13) == ((word32)rax)) { + goto L_AES_XTS_decrypt_avx512_done_dec; + } +L_AES_XTS_decrypt_avx512_last_31_start: + z4 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z7 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z4 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z4), 31)); + z7 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z7), _mm512_castsi512_si128(z4), x12, 0x78)); + rcx = (word64)(rdi + r13); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z7))); + /* aes_dec_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf33 = (word32)r10; + zf34 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_XTS_decrypt_avx512_last_31_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf35 = (word32)r10; + zf36 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf35) < (sword32)(zf36)) { + goto L_AES_XTS_decrypt_avx512_last_31_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_avx512_last_31_aes_dec_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesdeclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z7))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm512_castsi512_si128(z0)); + r13 = (word64)(r13 + 0x10); + rdx = (word64)(0); +L_AES_XTS_decrypt_avx512_last_31_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r13)) & 0xff); + WC_S8(rsi, r13) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r13 = (word32)((word32)r13 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r13) < ((word32)rax)) { + goto L_AES_XTS_decrypt_avx512_last_31_byte_loop; + } + r13 = (word64)(r13 - rdx); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + /* aes_dec_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r8, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 16))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 32))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 48))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 64))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 80))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 96))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf37 = (word32)r10; + zf38 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 160))); + if ((sword32)(zf37) < (sword32)(zf38)) { + goto L_AES_XTS_decrypt_avx512_last_31_2_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf39 = (word32)r10; + zf40 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 192))); + if ((sword32)(zf39) < (sword32)(zf40)) { + goto L_AES_XTS_decrypt_avx512_last_31_2_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, + 224))); +L_AES_XTS_decrypt_avx512_last_31_2_aes_dec_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesdeclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + r13 = (word64)(r13 - 0x10); + rcx = (word64)(rsi + r13); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); +L_AES_XTS_decrypt_avx512_done_dec: + ; +} + +WC_X64I_TARGET("aes,vaes,pclmul,vpclmulqdq,avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void AES_XTS_decrypt_update_avx512(const unsigned char* in, + unsigned char* out, unsigned int nbytes, const unsigned char* key, + unsigned char* tweak_block, int nr) +{ + word64 rdi, rsi, rax, r10, r8, r9, rsp, r12, r11, rcx = 0, rdx = 0; + __m128i x12; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), z8, + z9 = _mm512_setzero_si512(), z10 = _mm512_setzero_si512(), z13, + z14, z15, z16 = _mm512_setzero_si512(), + z17 = _mm512_setzero_si512(), z18 = _mm512_setzero_si512(), + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(), + z23 = _mm512_setzero_si512(), z24 = _mm512_setzero_si512(), + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(), + z27 = _mm512_setzero_si512(), z28 = _mm512_setzero_si512(), + z29 = _mm512_setzero_si512(), z30 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[8]; + word32 zf1; + word32 zf2; + word32 zf3; + word32 zf4; + word32 zf5; + word32 zf6; + word32 zf7; + word32 zf8; + word32 zf9; + word32 zf10; + word32 zf11; + word32 zf12; + word32 zf13; + word32 zf14; + word32 zf15; + word32 zf16; + word32 zf17; + word32 zf18; + word32 zf19; + word32 zf20; + word32 zf21; + word32 zf22; + word32 zf23; + word32 zf24; + word32 zf25; + word32 zf26; + word32 zf27; + word32 zf28; + word32 zf29; + word32 zf30; + word32 zf31; + word32 zf32; + word32 zf33; + word32 zf34; + word32 zf35; + word32 zf36; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rax = (word64)(word32)nbytes; + r10 = (word64)(size_t)key; + r8 = (word64)(size_t)tweak_block; + r9 = (word64)(word32)nr; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx512_aes_xts_gc_xts, 0)); + z13 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR( + L_avx512_aes_xts_poly, 0))); + z14 = _mm512_loadu_si512((const void*)WC_PR(L_avx512_aes_xts_shl, 0)); + z15 = _mm512_loadu_si512((const void*)WC_PR(L_avx512_aes_xts_shr, 0)); + z8 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r8, 0))); + r12 = (word32)(0); + r11 = (word32)((word32)rax); + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx512_mul16_256; + } + r11 = (word32)((word32)r11 - 0x10); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_start; + } +L_AES_XTS_decrypt_update_avx512_mul16_256: + if (((word32)r11) < (0x20)) { + goto L_AES_XTS_decrypt_update_avx512_done_128; + } + z16 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 0))); + z17 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + z18 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + z19 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + z20 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + z21 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + z22 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + z23 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + z24 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + z25 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + z26 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)((word32)r9) < (sword32)(0xb)) { + goto L_AES_XTS_decrypt_update_avx512_key_cached; + } + z27 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + z28 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)((word32)r9) < (sword32)(0xd)) { + goto L_AES_XTS_decrypt_update_avx512_key_cached; + } + z29 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + z30 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_avx512_key_cached: + if (((word32)r11) < (0x100)) { + goto L_AES_XTS_decrypt_update_avx512_done_256; + } + r11 = (word32)((word32)r11 & 0xffffff00); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + z9 = _mm512_srli_epi64(z4, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z4, 4); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + z9 = _mm512_srli_epi64(z5, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_slli_epi64(z5, 4); + z6 = _mm512_ternarylogic_epi64(z6, z10, z9, 0x96); + z9 = _mm512_srli_epi64(z6, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_slli_epi64(z6, 4); + z7 = _mm512_ternarylogic_epi64(z7, z10, z9, 0x96); +L_AES_XTS_decrypt_update_avx512_dec_256: + /* 256 bytes of input */ + /* aes_dec_256 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + /* aes_dec_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z1 = _mm512_ternarylogic_epi64(z1, z16, z5, 0x96); + z2 = _mm512_ternarylogic_epi64(z2, z16, z6, 0x96); + z3 = _mm512_ternarylogic_epi64(z3, z16, z7, 0x96); + z0 = _mm512_aesdec_epi128(z0, z17); + z1 = _mm512_aesdec_epi128(z1, z17); + z2 = _mm512_aesdec_epi128(z2, z17); + z3 = _mm512_aesdec_epi128(z3, z17); + z0 = _mm512_aesdec_epi128(z0, z18); + z1 = _mm512_aesdec_epi128(z1, z18); + z2 = _mm512_aesdec_epi128(z2, z18); + z3 = _mm512_aesdec_epi128(z3, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z1 = _mm512_aesdec_epi128(z1, z19); + z2 = _mm512_aesdec_epi128(z2, z19); + z3 = _mm512_aesdec_epi128(z3, z19); + z0 = _mm512_aesdec_epi128(z0, z20); + z1 = _mm512_aesdec_epi128(z1, z20); + z2 = _mm512_aesdec_epi128(z2, z20); + z3 = _mm512_aesdec_epi128(z3, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z1 = _mm512_aesdec_epi128(z1, z21); + z2 = _mm512_aesdec_epi128(z2, z21); + z3 = _mm512_aesdec_epi128(z3, z21); + z0 = _mm512_aesdec_epi128(z0, z22); + z1 = _mm512_aesdec_epi128(z1, z22); + z2 = _mm512_aesdec_epi128(z2, z22); + z3 = _mm512_aesdec_epi128(z3, z22); + z0 = _mm512_aesdec_epi128(z0, z23); + z1 = _mm512_aesdec_epi128(z1, z23); + z2 = _mm512_aesdec_epi128(z2, z23); + z3 = _mm512_aesdec_epi128(z3, z23); + z0 = _mm512_aesdec_epi128(z0, z24); + z1 = _mm512_aesdec_epi128(z1, z24); + z2 = _mm512_aesdec_epi128(z2, z24); + z3 = _mm512_aesdec_epi128(z3, z24); + z0 = _mm512_aesdec_epi128(z0, z25); + z1 = _mm512_aesdec_epi128(z1, z25); + z2 = _mm512_aesdec_epi128(z2, z25); + z3 = _mm512_aesdec_epi128(z3, z25); + zf1 = (word32)r9; + zf2 = 0xb; + z9 = z26; + if ((sword32)(zf1) < (sword32)(zf2)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_256_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z26); + z1 = _mm512_aesdec_epi128(z1, z26); + z2 = _mm512_aesdec_epi128(z2, z26); + z3 = _mm512_aesdec_epi128(z3, z26); + z0 = _mm512_aesdec_epi128(z0, z27); + z1 = _mm512_aesdec_epi128(z1, z27); + z2 = _mm512_aesdec_epi128(z2, z27); + z3 = _mm512_aesdec_epi128(z3, z27); + zf3 = (word32)r9; + zf4 = 0xd; + z9 = z28; + if ((sword32)(zf3) < (sword32)(zf4)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_256_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z28); + z1 = _mm512_aesdec_epi128(z1, z28); + z2 = _mm512_aesdec_epi128(z2, z28); + z3 = _mm512_aesdec_epi128(z3, z28); + z0 = _mm512_aesdec_epi128(z0, z29); + z1 = _mm512_aesdec_epi128(z1, z29); + z2 = _mm512_aesdec_epi128(z2, z29); + z3 = _mm512_aesdec_epi128(z3, z29); + z9 = z30; +L_AES_XTS_decrypt_update_avx512_aes_dec_256_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z9); + z1 = _mm512_aesdeclast_epi128(z1, z9); + z2 = _mm512_aesdeclast_epi128(z2, z9); + z3 = _mm512_aesdeclast_epi128(z3, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z9 = _mm512_srli_epi64(z4, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z4 = _mm512_slli_epi64(z4, 16); + z4 = _mm512_ternarylogic_epi64(z4, z10, z9, 0x96); + z1 = _mm512_xor_si512(z1, z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + z9 = _mm512_srli_epi64(z5, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z5, 16); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + z2 = _mm512_xor_si512(z2, z6); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z2); + z9 = _mm512_srli_epi64(z6, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z6 = _mm512_slli_epi64(z6, 16); + z6 = _mm512_ternarylogic_epi64(z6, z10, z9, 0x96); + z3 = _mm512_xor_si512(z3, z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z3); + z9 = _mm512_srli_epi64(z7, 48); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z7 = _mm512_slli_epi64(z7, 16); + z7 = _mm512_ternarylogic_epi64(z7, z10, z9, 0x96); + r12 = (word32)((word32)r12 + 0x100); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_decrypt_update_avx512_dec_256; + } + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 0)); +L_AES_XTS_decrypt_update_avx512_done_256: + zf5 = (word32)r12; + zf6 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf5) == (zf6)) { + goto L_AES_XTS_decrypt_update_avx512_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx512_mul16_128; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r12); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r12); +L_AES_XTS_decrypt_update_avx512_mul16_128: + r11 = (word32)((word32)r11 & 0xffffff80); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_decrypt_update_avx512_done_128; + } + /* 128 bytes of input */ + /* aes_dec_128 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + z9 = _mm512_srli_epi64(z4, 60); + z10 = _mm512_clmulepi64_epi128(z9, z13, 1); + z9 = _mm512_bslli_epi128(z9, 8); + z5 = _mm512_slli_epi64(z4, 4); + z5 = _mm512_ternarylogic_epi64(z5, z10, z9, 0x96); + /* aes_dec_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z1 = _mm512_ternarylogic_epi64(z1, z16, z5, 0x96); + z0 = _mm512_aesdec_epi128(z0, z17); + z1 = _mm512_aesdec_epi128(z1, z17); + z0 = _mm512_aesdec_epi128(z0, z18); + z1 = _mm512_aesdec_epi128(z1, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z1 = _mm512_aesdec_epi128(z1, z19); + z0 = _mm512_aesdec_epi128(z0, z20); + z1 = _mm512_aesdec_epi128(z1, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z1 = _mm512_aesdec_epi128(z1, z21); + z0 = _mm512_aesdec_epi128(z0, z22); + z1 = _mm512_aesdec_epi128(z1, z22); + z0 = _mm512_aesdec_epi128(z0, z23); + z1 = _mm512_aesdec_epi128(z1, z23); + z0 = _mm512_aesdec_epi128(z0, z24); + z1 = _mm512_aesdec_epi128(z1, z24); + z0 = _mm512_aesdec_epi128(z0, z25); + z1 = _mm512_aesdec_epi128(z1, z25); + zf7 = (word32)r9; + zf8 = 0xb; + z9 = z26; + if ((sword32)(zf7) < (sword32)(zf8)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_128_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z26); + z1 = _mm512_aesdec_epi128(z1, z26); + z0 = _mm512_aesdec_epi128(z0, z27); + z1 = _mm512_aesdec_epi128(z1, z27); + zf9 = (word32)r9; + zf10 = 0xd; + z9 = z28; + if ((sword32)(zf9) < (sword32)(zf10)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_128_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z28); + z1 = _mm512_aesdec_epi128(z1, z28); + z0 = _mm512_aesdec_epi128(z0, z29); + z1 = _mm512_aesdec_epi128(z1, z29); + z9 = z30; +L_AES_XTS_decrypt_update_avx512_aes_dec_128_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z9); + z1 = _mm512_aesdeclast_epi128(z1, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z1 = _mm512_xor_si512(z1, z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 3)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z9 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z9), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z9), x12, 0x78)); + r12 = (word32)((word32)r12 + 0x80); +L_AES_XTS_decrypt_update_avx512_done_128: + zf11 = (word32)r12; + zf12 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf11) == (zf12)) { + goto L_AES_XTS_decrypt_update_avx512_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx512_mul16_64; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r12); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r12); +L_AES_XTS_decrypt_update_avx512_mul16_64: + r11 = (word32)((word32)r11 & 0xffffffc0); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_decrypt_update_avx512_done_64; + } + /* 64 bytes of input */ + /* aes_dec_64 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z0 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + /* aes_dec_block */ + z0 = _mm512_ternarylogic_epi64(z0, z16, z4, 0x96); + z0 = _mm512_aesdec_epi128(z0, z17); + z0 = _mm512_aesdec_epi128(z0, z18); + z0 = _mm512_aesdec_epi128(z0, z19); + z0 = _mm512_aesdec_epi128(z0, z20); + z0 = _mm512_aesdec_epi128(z0, z21); + z0 = _mm512_aesdec_epi128(z0, z22); + z0 = _mm512_aesdec_epi128(z0, z23); + z0 = _mm512_aesdec_epi128(z0, z24); + z0 = _mm512_aesdec_epi128(z0, z25); + zf13 = (word32)r9; + zf14 = 0xb; + z9 = z26; + if ((sword32)(zf13) < (sword32)(zf14)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_64_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z26); + z0 = _mm512_aesdec_epi128(z0, z27); + zf15 = (word32)r9; + zf16 = 0xd; + z9 = z28; + if ((sword32)(zf15) < (sword32)(zf16)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_64_aes_dec_block_last; + } + z0 = _mm512_aesdec_epi128(z0, z28); + z0 = _mm512_aesdec_epi128(z0, z29); + z9 = z30; +L_AES_XTS_decrypt_update_avx512_aes_dec_64_aes_dec_block_last: + z0 = _mm512_aesdeclast_epi128(z0, z9); + z0 = _mm512_xor_si512(z0, z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 3)); + z9 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z9 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z9), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z9), x12, 0x78)); + r12 = (word32)((word32)r12 + 0x40); +L_AES_XTS_decrypt_update_avx512_done_64: + zf17 = (word32)r12; + zf18 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf17) == (zf18)) { + goto L_AES_XTS_decrypt_update_avx512_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx512_mul16_32; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r12); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r12); +L_AES_XTS_decrypt_update_avx512_mul16_32: + r11 = (word32)((word32)r11 & 0xffffffe0); + if (((word32)r12) == ((word32)r11)) { + goto L_AES_XTS_decrypt_update_avx512_done_32; + } + /* 32 bytes of input */ + /* aes_dec_32 */ + rcx = (word64)(rdi + r12); + rdx = (word64)(rsi + r12); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z5 = _mm512_shuffle_i64x2(z8, z8, 0); + z6 = _mm512_srlv_epi64(z5, z15); + z7 = _mm512_clmulepi64_epi128(z6, z13, 1); + z6 = _mm512_bslli_epi128(z6, 8); + z4 = _mm512_sllv_epi64(z5, z14); + z4 = _mm512_ternarylogic_epi64(z4, z7, z6, 0x96); + /* aes_dec_block */ + z0 = _mm512_zextsi256_si512(_mm256_ternarylogic_epi64( + _mm512_castsi512_si256(z0), _mm512_castsi512_si256(z16), + _mm512_castsi512_si256(z4), 0x96)); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z17))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z18))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z19))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z20))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z21))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z22))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z23))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z24))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z25))); + zf19 = (word32)r9; + zf20 = 0xb; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z26)); + if ((sword32)(zf19) < (sword32)(zf20)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_32_aes_dec_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z26))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z27))); + zf21 = (word32)r9; + zf22 = 0xd; + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z28)); + if ((sword32)(zf21) < (sword32)(zf22)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_32_aes_dec_block_last; + } + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z28))); + z0 = _mm512_zextsi256_si512(_mm256_aesdec_epi128(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z29))); + z9 = _mm512_zextsi256_si512(_mm512_castsi512_si256(z30)); +L_AES_XTS_decrypt_update_avx512_aes_dec_32_aes_dec_block_last: + z0 = _mm512_zextsi256_si512(_mm256_aesdeclast_epi128(_mm512_castsi512_si256( + z0), _mm512_castsi512_si256(z9))); + z0 = _mm512_zextsi256_si512(_mm256_xor_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z4))); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + z8 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + r12 = (word32)((word32)r12 + 0x20); +L_AES_XTS_decrypt_update_avx512_done_32: + zf23 = (word32)r12; + zf24 = (word32)rax; + r11 = (word32)((word32)rax); + if ((zf23) == (zf24)) { + goto L_AES_XTS_decrypt_update_avx512_done_dec; + } + r11 = (word32)((word32)r11 & 0xfffffff0); + if (((word32)r11) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx512_mul16; + } + r11 = (word32)((word32)r11 - 0x10); + r11 = (word32)((word32)r11 - (word32)r12); + if (((word32)r11) < (0x10)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_start; + } + r11 = (word32)((word32)r11 + (word32)r12); +L_AES_XTS_decrypt_update_avx512_mul16: +L_AES_XTS_decrypt_update_avx512_dec_16: + /* 16 bytes of input */ + rcx = (word64)(rdi + r12); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + /* aes_dec_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf25 = (word32)r9; + zf26 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf25) < (sword32)(zf26)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf27 = (word32)r9; + zf28 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf27) < (sword32)(zf28)) { + goto L_AES_XTS_decrypt_update_avx512_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_avx512_aes_dec_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesdeclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + z4 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z8 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z4 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z4), 31)); + z8 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z8), _mm512_castsi512_si128(z4), x12, 0x78)); + r12 = (word32)((word32)r12 + 0x10); + if (((word32)r12) < ((word32)r11)) { + goto L_AES_XTS_decrypt_update_avx512_dec_16; + } + if (((word32)r12) == ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx512_done_dec; + } +L_AES_XTS_decrypt_update_avx512_last_31_start: + z4 = _mm512_zextsi128_si512(_mm_shuffle_epi32(_mm512_castsi512_si128(z8), + 0x13)); + z7 = _mm512_zextsi128_si512(_mm_add_epi64(_mm512_castsi512_si128(z8), + _mm512_castsi512_si128(z8))); + z4 = _mm512_zextsi128_si512(_mm_srai_epi32(_mm512_castsi512_si128(z4), 31)); + z7 = _mm512_zextsi128_si512(_mm_ternarylogic_epi32(_mm512_castsi512_si128( + z7), _mm512_castsi512_si128(z4), x12, 0x78)); + rcx = (word64)(rdi + r12); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z7))); + /* aes_dec_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf29 = (word32)r9; + zf30 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf29) < (sword32)(zf30)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf31 = (word32)r9; + zf32 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf31) < (sword32)(zf32)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_avx512_last_31_aes_dec_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesdeclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z7))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm512_castsi512_si128(z0)); + r12 = (word64)(r12 + 0x10); + rdx = (word64)(0); +L_AES_XTS_decrypt_update_avx512_last_31_byte_loop: + r11 = (r11 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, rdx)) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, r12)) & 0xff); + WC_S8(rsi, r12) = (byte)((byte)r11); + WC_S8(rsp, rdx) = (byte)((byte)rcx); + r12 = (word32)((word32)r12 + 1); + rdx = (word32)((word32)rdx + 1); + if (((word32)r12) < ((word32)rax)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_byte_loop; + } + r12 = (word64)(r12 - rdx); + z0 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(rsp, 0))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + /* aes_dec_block */ + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 16))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 32))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 48))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 64))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 80))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 96))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 112))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 128))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 144))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + zf33 = (word32)r9; + zf34 = 0xb; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 160))); + if ((sword32)(zf33) < (sword32)(zf34)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_2_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 176))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + zf35 = (word32)r9; + zf36 = 0xd; + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 192))); + if ((sword32)(zf35) < (sword32)(zf36)) { + goto L_AES_XTS_decrypt_update_avx512_last_31_2_aes_dec_block_last; + } + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z6 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 208))); + z0 = _mm512_zextsi128_si512(_mm_aesdec_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z6))); + z5 = _mm512_zextsi128_si512(_mm_loadu_si128((const __m128i*)WC_PR(r10, + 224))); +L_AES_XTS_decrypt_update_avx512_last_31_2_aes_dec_block_last: + z0 = _mm512_zextsi128_si512(_mm_aesdeclast_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z5))); + z0 = _mm512_zextsi128_si512(_mm_xor_si128(_mm512_castsi512_si128(z0), + _mm512_castsi512_si128(z8))); + r12 = (word64)(r12 - 0x10); + rcx = (word64)(rsi + r12); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); +L_AES_XTS_decrypt_update_avx512_done_dec: + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z8)); +} + +#endif /* HAVE_INTEL_AVX512 */ +#endif /* WOLFSSL_X86_64_BUILD */ +#endif /* WOLFSSL_AES_XTS */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/chacha_intrin.c b/wolfcrypt/src/chacha_intrin.c new file mode 100644 index 00000000000..341d94db21a --- /dev/null +++ b/wolfcrypt/src/chacha_intrin.c @@ -0,0 +1,11594 @@ +/* chacha_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define _WC_BUILDING_CHACHA_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_X86_64_BUILD +extern WOLFSSL_LOCAL void chacha_encrypt_x64(ChaCha* ctx, const byte* m, + byte* c, word32 bytes); +#ifdef HAVE_INTEL_SSSE3 +extern WOLFSSL_LOCAL void chacha_encrypt_sse3(ChaCha* ctx, const byte* m, + byte* c, word32 bytes); +#endif +#ifdef HAVE_INTEL_AVX1 +extern WOLFSSL_LOCAL void chacha_encrypt_avx1(ChaCha* ctx, const byte* m, + byte* c, word32 bytes); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void chacha_encrypt_avx2(ChaCha* ctx, const byte* m, + byte* c, word32 bytes); +#endif +#ifdef HAVE_INTEL_AVX512 +extern WOLFSSL_LOCAL void chacha_encrypt_avx512vl(ChaCha* ctx, const byte* m, + byte* c, word32 bytes); +extern WOLFSSL_LOCAL void chacha_encrypt_avx512(ChaCha* ctx, const byte* m, + byte* c, word32 bytes); +extern WOLFSSL_LOCAL void chacha20_poly1305_avx512(ChaCha* chacha, + Poly1305* poly, const byte* m, byte* c, word32 bytes); +extern WOLFSSL_LOCAL void chacha20_poly1305_ifma(ChaCha* chacha, Poly1305* poly, + const byte* m, byte* c, word32 bytes); +extern WOLFSSL_LOCAL void chacha20_poly1305_ifma_decrypt(ChaCha* chacha, + Poly1305* poly, const byte* m, byte* c, word32 bytes); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void chacha20_poly1305_small_enc(ChaCha* chacha, + Poly1305* poly, const byte* m, byte* c, word32 mLen, const byte* aad, + word32 aadLen, byte* tag); +extern WOLFSSL_LOCAL int chacha20_poly1305_small_dec(ChaCha* chacha, + Poly1305* poly, const byte* in, byte* out, word32 ctLen, const byte* aad, + word32 aadLen, const byte* tag); + +#endif +#endif +#ifdef WOLFSSL_X86_64_BUILD +WOLFSSL_LOCAL void chacha_encrypt_x64(ChaCha* ctx, const byte* m, byte* c, + word32 bytes) +{ + word64 rdi, rsi, rdx, rcx, rsp, rax = 0, rbx = 0, r8 = 0, r9 = 0, r10 = 0, + r11 = 0, r12 = 0, r13 = 0, r14 = 0, r15 = 0, rbp = 0; + XALIGNED(32) WC_X64I_SLOT stk[16]; + word32 zf1; + word32 zf2; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rdx = (word64)(size_t)c; + rcx = (word64)(word32)bytes; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 64); + if (((word32)rcx) < (0x40)) { + goto L_chacha_x64_small; + } +L_chacha_x64_start: + rsp = (word64)(rsp - 48); + WC_S64(rsp, 24) = (word64)(rdx); + WC_S64(rsp, 32) = (word64)(rsi); + WC_S64(rsp, 40) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 32)); + rbx = (word64)(WC_L64(rdi, 40)); + WC_S64(rsp, 8) = (word64)(rax); + WC_S64(rsp, 16) = (word64)(rbx); + rax = (word32)(WC_L32(rdi, 0)); + rbx = (word32)(WC_L32(rdi, 4)); + rcx = (word32)(WC_L32(rdi, 8)); + rdx = (word32)(WC_L32(rdi, 12)); + r8 = (word32)(WC_L32(rdi, 16)); + r9 = (word32)(WC_L32(rdi, 20)); + r10 = (word32)(WC_L32(rdi, 24)); + r11 = (word32)(WC_L32(rdi, 28)); + r12 = (word32)(WC_L32(rdi, 48)); + r13 = (word32)(WC_L32(rdi, 52)); + r14 = (word32)(WC_L32(rdi, 56)); + r15 = (word32)(WC_L32(rdi, 60)); + WC_S8(rsp, 0) = (byte)(0xa); +L_chacha_x64_block_crypt_start: + rax = (word32)((word32)rax + (word32)r8); + rbx = (word32)((word32)rbx + (word32)r9); + rcx = (word32)((word32)rcx + (word32)r10); + rdx = (word32)((word32)rdx + (word32)r11); + r12 = (word32)((word32)r12 ^ (word32)rax); + r13 = (word32)((word32)r13 ^ (word32)rbx); + r14 = (word32)((word32)r14 ^ (word32)rcx); + r15 = (word32)((word32)r15 ^ (word32)rdx); + r12 = (word32)((((word32)r12) << 16) | (((word32)r12) >> 16)); + r13 = (word32)((((word32)r13) << 16) | (((word32)r13) >> 16)); + r14 = (word32)((((word32)r14) << 16) | (((word32)r14) >> 16)); + r15 = (word32)((((word32)r15) << 16) | (((word32)r15) >> 16)); + WC_S32(rsp, 8) = (word32)(WC_L32(rsp, 8) + (word32)r12); + WC_S32(rsp, 12) = (word32)(WC_L32(rsp, 12) + (word32)r13); + WC_S32(rsp, 16) = (word32)(WC_L32(rsp, 16) + (word32)r14); + WC_S32(rsp, 20) = (word32)(WC_L32(rsp, 20) + (word32)r15); + r8 = (word32)((word32)r8 ^ WC_L32(rsp, 8)); + r9 = (word32)((word32)r9 ^ WC_L32(rsp, 12)); + r10 = (word32)((word32)r10 ^ WC_L32(rsp, 16)); + r11 = (word32)((word32)r11 ^ WC_L32(rsp, 20)); + r8 = (word32)((((word32)r8) << 12) | (((word32)r8) >> 20)); + r9 = (word32)((((word32)r9) << 12) | (((word32)r9) >> 20)); + r10 = (word32)((((word32)r10) << 12) | (((word32)r10) >> 20)); + r11 = (word32)((((word32)r11) << 12) | (((word32)r11) >> 20)); + rax = (word32)((word32)rax + (word32)r8); + rbx = (word32)((word32)rbx + (word32)r9); + rcx = (word32)((word32)rcx + (word32)r10); + rdx = (word32)((word32)rdx + (word32)r11); + r12 = (word32)((word32)r12 ^ (word32)rax); + r13 = (word32)((word32)r13 ^ (word32)rbx); + r14 = (word32)((word32)r14 ^ (word32)rcx); + r15 = (word32)((word32)r15 ^ (word32)rdx); + r12 = (word32)((((word32)r12) << 8) | (((word32)r12) >> 24)); + r13 = (word32)((((word32)r13) << 8) | (((word32)r13) >> 24)); + r14 = (word32)((((word32)r14) << 8) | (((word32)r14) >> 24)); + r15 = (word32)((((word32)r15) << 8) | (((word32)r15) >> 24)); + WC_S32(rsp, 8) = (word32)(WC_L32(rsp, 8) + (word32)r12); + WC_S32(rsp, 12) = (word32)(WC_L32(rsp, 12) + (word32)r13); + WC_S32(rsp, 16) = (word32)(WC_L32(rsp, 16) + (word32)r14); + WC_S32(rsp, 20) = (word32)(WC_L32(rsp, 20) + (word32)r15); + r8 = (word32)((word32)r8 ^ WC_L32(rsp, 8)); + r9 = (word32)((word32)r9 ^ WC_L32(rsp, 12)); + r10 = (word32)((word32)r10 ^ WC_L32(rsp, 16)); + r11 = (word32)((word32)r11 ^ WC_L32(rsp, 20)); + r8 = (word32)((((word32)r8) << 7) | (((word32)r8) >> 25)); + r9 = (word32)((((word32)r9) << 7) | (((word32)r9) >> 25)); + r10 = (word32)((((word32)r10) << 7) | (((word32)r10) >> 25)); + r11 = (word32)((((word32)r11) << 7) | (((word32)r11) >> 25)); + rax = (word32)((word32)rax + (word32)r9); + rbx = (word32)((word32)rbx + (word32)r10); + rcx = (word32)((word32)rcx + (word32)r11); + rdx = (word32)((word32)rdx + (word32)r8); + r15 = (word32)((word32)r15 ^ (word32)rax); + r12 = (word32)((word32)r12 ^ (word32)rbx); + r13 = (word32)((word32)r13 ^ (word32)rcx); + r14 = (word32)((word32)r14 ^ (word32)rdx); + r15 = (word32)((((word32)r15) << 16) | (((word32)r15) >> 16)); + r12 = (word32)((((word32)r12) << 16) | (((word32)r12) >> 16)); + r13 = (word32)((((word32)r13) << 16) | (((word32)r13) >> 16)); + r14 = (word32)((((word32)r14) << 16) | (((word32)r14) >> 16)); + WC_S32(rsp, 16) = (word32)(WC_L32(rsp, 16) + (word32)r15); + WC_S32(rsp, 20) = (word32)(WC_L32(rsp, 20) + (word32)r12); + WC_S32(rsp, 8) = (word32)(WC_L32(rsp, 8) + (word32)r13); + WC_S32(rsp, 12) = (word32)(WC_L32(rsp, 12) + (word32)r14); + r9 = (word32)((word32)r9 ^ WC_L32(rsp, 16)); + r10 = (word32)((word32)r10 ^ WC_L32(rsp, 20)); + r11 = (word32)((word32)r11 ^ WC_L32(rsp, 8)); + r8 = (word32)((word32)r8 ^ WC_L32(rsp, 12)); + r9 = (word32)((((word32)r9) << 12) | (((word32)r9) >> 20)); + r10 = (word32)((((word32)r10) << 12) | (((word32)r10) >> 20)); + r11 = (word32)((((word32)r11) << 12) | (((word32)r11) >> 20)); + r8 = (word32)((((word32)r8) << 12) | (((word32)r8) >> 20)); + rax = (word32)((word32)rax + (word32)r9); + rbx = (word32)((word32)rbx + (word32)r10); + rcx = (word32)((word32)rcx + (word32)r11); + rdx = (word32)((word32)rdx + (word32)r8); + r15 = (word32)((word32)r15 ^ (word32)rax); + r12 = (word32)((word32)r12 ^ (word32)rbx); + r13 = (word32)((word32)r13 ^ (word32)rcx); + r14 = (word32)((word32)r14 ^ (word32)rdx); + r15 = (word32)((((word32)r15) << 8) | (((word32)r15) >> 24)); + r12 = (word32)((((word32)r12) << 8) | (((word32)r12) >> 24)); + r13 = (word32)((((word32)r13) << 8) | (((word32)r13) >> 24)); + r14 = (word32)((((word32)r14) << 8) | (((word32)r14) >> 24)); + WC_S32(rsp, 16) = (word32)(WC_L32(rsp, 16) + (word32)r15); + WC_S32(rsp, 20) = (word32)(WC_L32(rsp, 20) + (word32)r12); + WC_S32(rsp, 8) = (word32)(WC_L32(rsp, 8) + (word32)r13); + WC_S32(rsp, 12) = (word32)(WC_L32(rsp, 12) + (word32)r14); + r9 = (word32)((word32)r9 ^ WC_L32(rsp, 16)); + r10 = (word32)((word32)r10 ^ WC_L32(rsp, 20)); + r11 = (word32)((word32)r11 ^ WC_L32(rsp, 8)); + r8 = (word32)((word32)r8 ^ WC_L32(rsp, 12)); + r9 = (word32)((((word32)r9) << 7) | (((word32)r9) >> 25)); + r10 = (word32)((((word32)r10) << 7) | (((word32)r10) >> 25)); + r11 = (word32)((((word32)r11) << 7) | (((word32)r11) >> 25)); + r8 = (word32)((((word32)r8) << 7) | (((word32)r8) >> 25)); + WC_S8(rsp, 0) = (byte)(WC_L8(rsp, 0) - 1); + if (((byte)WC_S8(rsp, 0)) != (0)) { + goto L_chacha_x64_block_crypt_start; + } + rsi = (word64)(WC_L64(rsp, 32)); + rbp = (word64)(WC_L64(rsp, 24)); + rax = (word32)((word32)rax + WC_L32(rdi, 0)); + rbx = (word32)((word32)rbx + WC_L32(rdi, 4)); + rcx = (word32)((word32)rcx + WC_L32(rdi, 8)); + rdx = (word32)((word32)rdx + WC_L32(rdi, 12)); + r8 = (word32)((word32)r8 + WC_L32(rdi, 16)); + r9 = (word32)((word32)r9 + WC_L32(rdi, 20)); + r10 = (word32)((word32)r10 + WC_L32(rdi, 24)); + r11 = (word32)((word32)r11 + WC_L32(rdi, 28)); + r12 = (word32)((word32)r12 + WC_L32(rdi, 48)); + r13 = (word32)((word32)r13 + WC_L32(rdi, 52)); + r14 = (word32)((word32)r14 + WC_L32(rdi, 56)); + r15 = (word32)((word32)r15 + WC_L32(rdi, 60)); + rax = (word32)((word32)rax ^ WC_L32(rsi, 0)); + rbx = (word32)((word32)rbx ^ WC_L32(rsi, 4)); + rcx = (word32)((word32)rcx ^ WC_L32(rsi, 8)); + rdx = (word32)((word32)rdx ^ WC_L32(rsi, 12)); + r8 = (word32)((word32)r8 ^ WC_L32(rsi, 16)); + r9 = (word32)((word32)r9 ^ WC_L32(rsi, 20)); + r10 = (word32)((word32)r10 ^ WC_L32(rsi, 24)); + r11 = (word32)((word32)r11 ^ WC_L32(rsi, 28)); + r12 = (word32)((word32)r12 ^ WC_L32(rsi, 48)); + r13 = (word32)((word32)r13 ^ WC_L32(rsi, 52)); + r14 = (word32)((word32)r14 ^ WC_L32(rsi, 56)); + r15 = (word32)((word32)r15 ^ WC_L32(rsi, 60)); + WC_S32(rbp, 0) = (word32)((word32)rax); + WC_S32(rbp, 4) = (word32)((word32)rbx); + WC_S32(rbp, 8) = (word32)((word32)rcx); + WC_S32(rbp, 12) = (word32)((word32)rdx); + WC_S32(rbp, 16) = (word32)((word32)r8); + WC_S32(rbp, 20) = (word32)((word32)r9); + WC_S32(rbp, 24) = (word32)((word32)r10); + WC_S32(rbp, 28) = (word32)((word32)r11); + WC_S32(rbp, 48) = (word32)((word32)r12); + WC_S32(rbp, 52) = (word32)((word32)r13); + WC_S32(rbp, 56) = (word32)((word32)r14); + WC_S32(rbp, 60) = (word32)((word32)r15); + rax = (word32)(WC_L32(rsp, 8)); + rbx = (word32)(WC_L32(rsp, 12)); + rcx = (word32)(WC_L32(rsp, 16)); + rdx = (word32)(WC_L32(rsp, 20)); + rax = (word32)((word32)rax + WC_L32(rdi, 32)); + rbx = (word32)((word32)rbx + WC_L32(rdi, 36)); + rcx = (word32)((word32)rcx + WC_L32(rdi, 40)); + rdx = (word32)((word32)rdx + WC_L32(rdi, 44)); + rax = (word32)((word32)rax ^ WC_L32(rsi, 32)); + rbx = (word32)((word32)rbx ^ WC_L32(rsi, 36)); + rcx = (word32)((word32)rcx ^ WC_L32(rsi, 40)); + rdx = (word32)((word32)rdx ^ WC_L32(rsi, 44)); + WC_S32(rbp, 32) = (word32)((word32)rax); + WC_S32(rbp, 36) = (word32)((word32)rbx); + WC_S32(rbp, 40) = (word32)((word32)rcx); + WC_S32(rbp, 44) = (word32)((word32)rdx); + rdx = (word64)(WC_L64(rsp, 24)); + rcx = (word64)(WC_L64(rsp, 40)); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 1); + rsp = (word64)(rsp + 48); + rcx = (word32)((word32)rcx - 0x40); + rsi = (word64)(rsi + 0x40); + rdx = (word64)(rdx + 0x40); + if (((word32)rcx) >= (0x40)) { + goto L_chacha_x64_start; + } +L_chacha_x64_small: + if (((word32)rcx) == (0)) { + goto L_chacha_x64_done; + } + rsp = (word64)(rsp - 48); + WC_S64(rsp, 24) = (word64)(rdx); + WC_S64(rsp, 32) = (word64)(rsi); + WC_S64(rsp, 40) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 32)); + rbx = (word64)(WC_L64(rdi, 40)); + WC_S64(rsp, 8) = (word64)(rax); + WC_S64(rsp, 16) = (word64)(rbx); + rax = (word32)(WC_L32(rdi, 0)); + rbx = (word32)(WC_L32(rdi, 4)); + rcx = (word32)(WC_L32(rdi, 8)); + rdx = (word32)(WC_L32(rdi, 12)); + r8 = (word32)(WC_L32(rdi, 16)); + r9 = (word32)(WC_L32(rdi, 20)); + r10 = (word32)(WC_L32(rdi, 24)); + r11 = (word32)(WC_L32(rdi, 28)); + r12 = (word32)(WC_L32(rdi, 48)); + r13 = (word32)(WC_L32(rdi, 52)); + r14 = (word32)(WC_L32(rdi, 56)); + r15 = (word32)(WC_L32(rdi, 60)); + WC_S8(rsp, 0) = (byte)(0xa); +L_chacha_x64_partial_crypt_start: + rax = (word32)((word32)rax + (word32)r8); + rbx = (word32)((word32)rbx + (word32)r9); + rcx = (word32)((word32)rcx + (word32)r10); + rdx = (word32)((word32)rdx + (word32)r11); + r12 = (word32)((word32)r12 ^ (word32)rax); + r13 = (word32)((word32)r13 ^ (word32)rbx); + r14 = (word32)((word32)r14 ^ (word32)rcx); + r15 = (word32)((word32)r15 ^ (word32)rdx); + r12 = (word32)((((word32)r12) << 16) | (((word32)r12) >> 16)); + r13 = (word32)((((word32)r13) << 16) | (((word32)r13) >> 16)); + r14 = (word32)((((word32)r14) << 16) | (((word32)r14) >> 16)); + r15 = (word32)((((word32)r15) << 16) | (((word32)r15) >> 16)); + WC_S32(rsp, 8) = (word32)(WC_L32(rsp, 8) + (word32)r12); + WC_S32(rsp, 12) = (word32)(WC_L32(rsp, 12) + (word32)r13); + WC_S32(rsp, 16) = (word32)(WC_L32(rsp, 16) + (word32)r14); + WC_S32(rsp, 20) = (word32)(WC_L32(rsp, 20) + (word32)r15); + r8 = (word32)((word32)r8 ^ WC_L32(rsp, 8)); + r9 = (word32)((word32)r9 ^ WC_L32(rsp, 12)); + r10 = (word32)((word32)r10 ^ WC_L32(rsp, 16)); + r11 = (word32)((word32)r11 ^ WC_L32(rsp, 20)); + r8 = (word32)((((word32)r8) << 12) | (((word32)r8) >> 20)); + r9 = (word32)((((word32)r9) << 12) | (((word32)r9) >> 20)); + r10 = (word32)((((word32)r10) << 12) | (((word32)r10) >> 20)); + r11 = (word32)((((word32)r11) << 12) | (((word32)r11) >> 20)); + rax = (word32)((word32)rax + (word32)r8); + rbx = (word32)((word32)rbx + (word32)r9); + rcx = (word32)((word32)rcx + (word32)r10); + rdx = (word32)((word32)rdx + (word32)r11); + r12 = (word32)((word32)r12 ^ (word32)rax); + r13 = (word32)((word32)r13 ^ (word32)rbx); + r14 = (word32)((word32)r14 ^ (word32)rcx); + r15 = (word32)((word32)r15 ^ (word32)rdx); + r12 = (word32)((((word32)r12) << 8) | (((word32)r12) >> 24)); + r13 = (word32)((((word32)r13) << 8) | (((word32)r13) >> 24)); + r14 = (word32)((((word32)r14) << 8) | (((word32)r14) >> 24)); + r15 = (word32)((((word32)r15) << 8) | (((word32)r15) >> 24)); + WC_S32(rsp, 8) = (word32)(WC_L32(rsp, 8) + (word32)r12); + WC_S32(rsp, 12) = (word32)(WC_L32(rsp, 12) + (word32)r13); + WC_S32(rsp, 16) = (word32)(WC_L32(rsp, 16) + (word32)r14); + WC_S32(rsp, 20) = (word32)(WC_L32(rsp, 20) + (word32)r15); + r8 = (word32)((word32)r8 ^ WC_L32(rsp, 8)); + r9 = (word32)((word32)r9 ^ WC_L32(rsp, 12)); + r10 = (word32)((word32)r10 ^ WC_L32(rsp, 16)); + r11 = (word32)((word32)r11 ^ WC_L32(rsp, 20)); + r8 = (word32)((((word32)r8) << 7) | (((word32)r8) >> 25)); + r9 = (word32)((((word32)r9) << 7) | (((word32)r9) >> 25)); + r10 = (word32)((((word32)r10) << 7) | (((word32)r10) >> 25)); + r11 = (word32)((((word32)r11) << 7) | (((word32)r11) >> 25)); + rax = (word32)((word32)rax + (word32)r9); + rbx = (word32)((word32)rbx + (word32)r10); + rcx = (word32)((word32)rcx + (word32)r11); + rdx = (word32)((word32)rdx + (word32)r8); + r15 = (word32)((word32)r15 ^ (word32)rax); + r12 = (word32)((word32)r12 ^ (word32)rbx); + r13 = (word32)((word32)r13 ^ (word32)rcx); + r14 = (word32)((word32)r14 ^ (word32)rdx); + r15 = (word32)((((word32)r15) << 16) | (((word32)r15) >> 16)); + r12 = (word32)((((word32)r12) << 16) | (((word32)r12) >> 16)); + r13 = (word32)((((word32)r13) << 16) | (((word32)r13) >> 16)); + r14 = (word32)((((word32)r14) << 16) | (((word32)r14) >> 16)); + WC_S32(rsp, 16) = (word32)(WC_L32(rsp, 16) + (word32)r15); + WC_S32(rsp, 20) = (word32)(WC_L32(rsp, 20) + (word32)r12); + WC_S32(rsp, 8) = (word32)(WC_L32(rsp, 8) + (word32)r13); + WC_S32(rsp, 12) = (word32)(WC_L32(rsp, 12) + (word32)r14); + r9 = (word32)((word32)r9 ^ WC_L32(rsp, 16)); + r10 = (word32)((word32)r10 ^ WC_L32(rsp, 20)); + r11 = (word32)((word32)r11 ^ WC_L32(rsp, 8)); + r8 = (word32)((word32)r8 ^ WC_L32(rsp, 12)); + r9 = (word32)((((word32)r9) << 12) | (((word32)r9) >> 20)); + r10 = (word32)((((word32)r10) << 12) | (((word32)r10) >> 20)); + r11 = (word32)((((word32)r11) << 12) | (((word32)r11) >> 20)); + r8 = (word32)((((word32)r8) << 12) | (((word32)r8) >> 20)); + rax = (word32)((word32)rax + (word32)r9); + rbx = (word32)((word32)rbx + (word32)r10); + rcx = (word32)((word32)rcx + (word32)r11); + rdx = (word32)((word32)rdx + (word32)r8); + r15 = (word32)((word32)r15 ^ (word32)rax); + r12 = (word32)((word32)r12 ^ (word32)rbx); + r13 = (word32)((word32)r13 ^ (word32)rcx); + r14 = (word32)((word32)r14 ^ (word32)rdx); + r15 = (word32)((((word32)r15) << 8) | (((word32)r15) >> 24)); + r12 = (word32)((((word32)r12) << 8) | (((word32)r12) >> 24)); + r13 = (word32)((((word32)r13) << 8) | (((word32)r13) >> 24)); + r14 = (word32)((((word32)r14) << 8) | (((word32)r14) >> 24)); + WC_S32(rsp, 16) = (word32)(WC_L32(rsp, 16) + (word32)r15); + WC_S32(rsp, 20) = (word32)(WC_L32(rsp, 20) + (word32)r12); + WC_S32(rsp, 8) = (word32)(WC_L32(rsp, 8) + (word32)r13); + WC_S32(rsp, 12) = (word32)(WC_L32(rsp, 12) + (word32)r14); + r9 = (word32)((word32)r9 ^ WC_L32(rsp, 16)); + r10 = (word32)((word32)r10 ^ WC_L32(rsp, 20)); + r11 = (word32)((word32)r11 ^ WC_L32(rsp, 8)); + r8 = (word32)((word32)r8 ^ WC_L32(rsp, 12)); + r9 = (word32)((((word32)r9) << 7) | (((word32)r9) >> 25)); + r10 = (word32)((((word32)r10) << 7) | (((word32)r10) >> 25)); + r11 = (word32)((((word32)r11) << 7) | (((word32)r11) >> 25)); + r8 = (word32)((((word32)r8) << 7) | (((word32)r8) >> 25)); + WC_S8(rsp, 0) = (byte)(WC_L8(rsp, 0) - 1); + if (((byte)WC_S8(rsp, 0)) != (0)) { + goto L_chacha_x64_partial_crypt_start; + } + rsi = (word64)(WC_L64(rsp, 32)); + rax = (word32)((word32)rax + WC_L32(rdi, 0)); + rbx = (word32)((word32)rbx + WC_L32(rdi, 4)); + rcx = (word32)((word32)rcx + WC_L32(rdi, 8)); + rdx = (word32)((word32)rdx + WC_L32(rdi, 12)); + r8 = (word32)((word32)r8 + WC_L32(rdi, 16)); + r9 = (word32)((word32)r9 + WC_L32(rdi, 20)); + r10 = (word32)((word32)r10 + WC_L32(rdi, 24)); + r11 = (word32)((word32)r11 + WC_L32(rdi, 28)); + r12 = (word32)((word32)r12 + WC_L32(rdi, 48)); + r13 = (word32)((word32)r13 + WC_L32(rdi, 52)); + r14 = (word32)((word32)r14 + WC_L32(rdi, 56)); + r15 = (word32)((word32)r15 + WC_L32(rdi, 60)); + rbp = (word64)(rdi + 80); + WC_S32(rbp, 0) = (word32)((word32)rax); + WC_S32(rbp, 4) = (word32)((word32)rbx); + WC_S32(rbp, 8) = (word32)((word32)rcx); + WC_S32(rbp, 12) = (word32)((word32)rdx); + WC_S32(rbp, 16) = (word32)((word32)r8); + WC_S32(rbp, 20) = (word32)((word32)r9); + WC_S32(rbp, 24) = (word32)((word32)r10); + WC_S32(rbp, 28) = (word32)((word32)r11); + WC_S32(rbp, 48) = (word32)((word32)r12); + WC_S32(rbp, 52) = (word32)((word32)r13); + WC_S32(rbp, 56) = (word32)((word32)r14); + WC_S32(rbp, 60) = (word32)((word32)r15); + rax = (word32)(WC_L32(rsp, 8)); + rbx = (word32)(WC_L32(rsp, 12)); + rcx = (word32)(WC_L32(rsp, 16)); + rdx = (word32)(WC_L32(rsp, 20)); + rax = (word32)((word32)rax + WC_L32(rdi, 32)); + rbx = (word32)((word32)rbx + WC_L32(rdi, 36)); + rcx = (word32)((word32)rcx + WC_L32(rdi, 40)); + rdx = (word32)((word32)rdx + WC_L32(rdi, 44)); + WC_S32(rbp, 32) = (word32)((word32)rax); + WC_S32(rbp, 36) = (word32)((word32)rbx); + WC_S32(rbp, 40) = (word32)((word32)rcx); + WC_S32(rbp, 44) = (word32)((word32)rdx); + rdx = (word64)(WC_L64(rsp, 24)); + rcx = (word64)(WC_L64(rsp, 40)); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 1); + rsp = (word64)(rsp + 48); + r8 = (word32)((word32)rcx); + rbx = (word64)(0); + r8 = (word32)((word32)r8 & 7); + if (((word32)r8) == (0)) { + goto L_chacha_x64_partial_start64; + } +L_chacha_x64_partial_start8: + rax = (word32)((word32)WC_L8(rbp, rbx)); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax ^ WC_L8(rsi, + rbx)) & 0xff); + WC_S8(rdx, rbx) = (byte)((byte)rax); + rbx = (word32)((word32)rbx + 1); + zf1 = (word32)rbx; + zf2 = (word32)r8; + if (((word32)rbx) != ((word32)r8)) { + goto L_chacha_x64_partial_start8; + } + if ((zf1) == (zf2)) { + goto L_chacha_x64_partial_end64; + } +L_chacha_x64_partial_start64: + rax = (word64)(WC_L64(rbp, rbx)); + rax = (word64)(rax ^ WC_L64(rsi, rbx)); + WC_S64(rdx, rbx) = (word64)(rax); + rbx = (word32)((word32)rbx + 8); +L_chacha_x64_partial_end64: + if (((word32)rbx) != ((word32)rcx)) { + goto L_chacha_x64_partial_start64; + } + rcx = (word32)(0x40); + rcx = (word32)((word32)rcx - (word32)rbx); + WC_S32(rdi, 76) = (word32)((word32)rcx); +L_chacha_x64_done: + ; +} + +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifdef HAVE_INTEL_SSSE3 +XALIGNED(16) static const word64 L_chacha20_sse3_rotl8[] WC_X64I_UNUSED = { + 0x0605040702010003ULL, 0x0e0d0c0f0a09080bULL, +}; + +XALIGNED(16) static const word64 L_chacha20_sse3_rotl16[] WC_X64I_UNUSED = { + 0x0504070601000302ULL, 0x0d0c0f0e09080b0aULL, +}; + +XALIGNED(16) static const word64 L_chacha20_sse3_one[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("sse2,ssse3") +WOLFSSL_LOCAL void chacha_encrypt_sse3(ChaCha* ctx, const byte* m, byte* c, + word32 bytes) +{ + word64 rdi, rsi, rdx, rcx, r11, r12, r13, rax = 0, r8 = 0, r9 = 0, + r10 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(); + word32 zf1; + word32 zf2; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rdx = (word64)(size_t)c; + rcx = (word64)(word32)bytes; + + r11 = (word64)((word64)(size_t)L_chacha20_sse3_rotl8); + r12 = (word64)((word64)(size_t)L_chacha20_sse3_rotl16); + r13 = (word64)((word64)(size_t)L_chacha20_sse3_one); + if (((word32)rcx) < (0x80)) { + goto L_chacha20_sse3_128_done; + } +L_chacha20_sse3_128_start: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x10 = x0; + x11 = x1; + x12 = x2; + x13 = x3; + x4 = x0; + x5 = x1; + x6 = x2; + x7 = x3; + x7 = _mm_add_epi32(x7, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + rax = (rax & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_chacha20_sse3_128_crypt2_start: + x0 = _mm_add_epi32(x0, x1); + x4 = _mm_add_epi32(x4, x5); + x3 = _mm_xor_si128(x3, x0); + x7 = _mm_xor_si128(x7, x4); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x2 = _mm_add_epi32(x2, x3); + x6 = _mm_add_epi32(x6, x7); + x1 = _mm_xor_si128(x1, x2); + x5 = _mm_xor_si128(x5, x6); + x8 = x1; + x9 = x5; + x8 = _mm_srli_epi32(x8, 20); + x9 = _mm_srli_epi32(x9, 20); + x1 = _mm_slli_epi32(x1, 12); + x5 = _mm_slli_epi32(x5, 12); + x1 = _mm_xor_si128(x1, x8); + x5 = _mm_xor_si128(x5, x9); + x0 = _mm_add_epi32(x0, x1); + x4 = _mm_add_epi32(x4, x5); + x3 = _mm_xor_si128(x3, x0); + x7 = _mm_xor_si128(x7, x4); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x2 = _mm_add_epi32(x2, x3); + x6 = _mm_add_epi32(x6, x7); + x1 = _mm_xor_si128(x1, x2); + x5 = _mm_xor_si128(x5, x6); + x8 = x1; + x9 = x5; + x8 = _mm_srli_epi32(x8, 25); + x9 = _mm_srli_epi32(x9, 25); + x1 = _mm_slli_epi32(x1, 7); + x5 = _mm_slli_epi32(x5, 7); + x1 = _mm_xor_si128(x1, x8); + x5 = _mm_xor_si128(x5, x9); + x1 = _mm_shuffle_epi32(x1, 0x39); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x3 = _mm_shuffle_epi32(x3, 0x93); + x5 = _mm_shuffle_epi32(x5, 0x39); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x7 = _mm_shuffle_epi32(x7, 0x93); + x0 = _mm_add_epi32(x0, x1); + x4 = _mm_add_epi32(x4, x5); + x3 = _mm_xor_si128(x3, x0); + x7 = _mm_xor_si128(x7, x4); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x2 = _mm_add_epi32(x2, x3); + x6 = _mm_add_epi32(x6, x7); + x1 = _mm_xor_si128(x1, x2); + x5 = _mm_xor_si128(x5, x6); + x8 = x1; + x9 = x5; + x8 = _mm_srli_epi32(x8, 20); + x9 = _mm_srli_epi32(x9, 20); + x1 = _mm_slli_epi32(x1, 12); + x5 = _mm_slli_epi32(x5, 12); + x1 = _mm_xor_si128(x1, x8); + x5 = _mm_xor_si128(x5, x9); + x0 = _mm_add_epi32(x0, x1); + x4 = _mm_add_epi32(x4, x5); + x3 = _mm_xor_si128(x3, x0); + x7 = _mm_xor_si128(x7, x4); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x2 = _mm_add_epi32(x2, x3); + x6 = _mm_add_epi32(x6, x7); + x1 = _mm_xor_si128(x1, x2); + x5 = _mm_xor_si128(x5, x6); + x8 = x1; + x9 = x5; + x8 = _mm_srli_epi32(x8, 25); + x9 = _mm_srli_epi32(x9, 25); + x1 = _mm_slli_epi32(x1, 7); + x5 = _mm_slli_epi32(x5, 7); + x1 = _mm_xor_si128(x1, x8); + x5 = _mm_xor_si128(x5, x9); + x1 = _mm_shuffle_epi32(x1, 0x93); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x3 = _mm_shuffle_epi32(x3, 0x39); + x5 = _mm_shuffle_epi32(x5, 0x93); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x7 = _mm_shuffle_epi32(x7, 0x39); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax - 1) & 0xff); + if (((byte)rax) != (0)) { + goto L_chacha20_sse3_128_crypt2_start; + } + x0 = _mm_add_epi32(x0, x10); + x1 = _mm_add_epi32(x1, x11); + x2 = _mm_add_epi32(x2, x12); + x3 = _mm_add_epi32(x3, x13); + x4 = _mm_add_epi32(x4, x10); + x5 = _mm_add_epi32(x5, x11); + x6 = _mm_add_epi32(x6, x12); + x7 = _mm_add_epi32(x7, x13); + x7 = _mm_add_epi32(x7, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_xor_si128(x0, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x1 = _mm_xor_si128(x1, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x1); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x2 = _mm_xor_si128(x2, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x2); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x3 = _mm_xor_si128(x3, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x3); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x4 = _mm_xor_si128(x4, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x4); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x5 = _mm_xor_si128(x5, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x5); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x6 = _mm_xor_si128(x6, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x6); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x7 = _mm_xor_si128(x7, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x7); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 2); + rcx = (word32)((word32)rcx - 0x80); + rsi = (word64)(rsi + 0x80); + rdx = (word64)(rdx + 0x80); + if (((word32)rcx) >= (0x80)) { + goto L_chacha20_sse3_128_start; + } +L_chacha20_sse3_128_done: + if (((word32)rcx) == (0)) { + goto L_chacha20_sse3_last_done; + } + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x10 = x0; + x11 = x1; + x12 = x2; + x13 = x3; + x4 = x0; + x5 = x1; + x6 = x2; + x7 = x3; + x7 = _mm_add_epi32(x7, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + rax = (rax & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_chacha20_sse3_last_crypt2_start: + x0 = _mm_add_epi32(x0, x1); + x4 = _mm_add_epi32(x4, x5); + x3 = _mm_xor_si128(x3, x0); + x7 = _mm_xor_si128(x7, x4); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x2 = _mm_add_epi32(x2, x3); + x6 = _mm_add_epi32(x6, x7); + x1 = _mm_xor_si128(x1, x2); + x5 = _mm_xor_si128(x5, x6); + x8 = x1; + x9 = x5; + x8 = _mm_srli_epi32(x8, 20); + x9 = _mm_srli_epi32(x9, 20); + x1 = _mm_slli_epi32(x1, 12); + x5 = _mm_slli_epi32(x5, 12); + x1 = _mm_xor_si128(x1, x8); + x5 = _mm_xor_si128(x5, x9); + x0 = _mm_add_epi32(x0, x1); + x4 = _mm_add_epi32(x4, x5); + x3 = _mm_xor_si128(x3, x0); + x7 = _mm_xor_si128(x7, x4); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x2 = _mm_add_epi32(x2, x3); + x6 = _mm_add_epi32(x6, x7); + x1 = _mm_xor_si128(x1, x2); + x5 = _mm_xor_si128(x5, x6); + x8 = x1; + x9 = x5; + x8 = _mm_srli_epi32(x8, 25); + x9 = _mm_srli_epi32(x9, 25); + x1 = _mm_slli_epi32(x1, 7); + x5 = _mm_slli_epi32(x5, 7); + x1 = _mm_xor_si128(x1, x8); + x5 = _mm_xor_si128(x5, x9); + x1 = _mm_shuffle_epi32(x1, 0x39); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x3 = _mm_shuffle_epi32(x3, 0x93); + x5 = _mm_shuffle_epi32(x5, 0x39); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x7 = _mm_shuffle_epi32(x7, 0x93); + x0 = _mm_add_epi32(x0, x1); + x4 = _mm_add_epi32(x4, x5); + x3 = _mm_xor_si128(x3, x0); + x7 = _mm_xor_si128(x7, x4); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x2 = _mm_add_epi32(x2, x3); + x6 = _mm_add_epi32(x6, x7); + x1 = _mm_xor_si128(x1, x2); + x5 = _mm_xor_si128(x5, x6); + x8 = x1; + x9 = x5; + x8 = _mm_srli_epi32(x8, 20); + x9 = _mm_srli_epi32(x9, 20); + x1 = _mm_slli_epi32(x1, 12); + x5 = _mm_slli_epi32(x5, 12); + x1 = _mm_xor_si128(x1, x8); + x5 = _mm_xor_si128(x5, x9); + x0 = _mm_add_epi32(x0, x1); + x4 = _mm_add_epi32(x4, x5); + x3 = _mm_xor_si128(x3, x0); + x7 = _mm_xor_si128(x7, x4); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x7 = _mm_shuffle_epi8(x7, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + x2 = _mm_add_epi32(x2, x3); + x6 = _mm_add_epi32(x6, x7); + x1 = _mm_xor_si128(x1, x2); + x5 = _mm_xor_si128(x5, x6); + x8 = x1; + x9 = x5; + x8 = _mm_srli_epi32(x8, 25); + x9 = _mm_srli_epi32(x9, 25); + x1 = _mm_slli_epi32(x1, 7); + x5 = _mm_slli_epi32(x5, 7); + x1 = _mm_xor_si128(x1, x8); + x5 = _mm_xor_si128(x5, x9); + x1 = _mm_shuffle_epi32(x1, 0x93); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x3 = _mm_shuffle_epi32(x3, 0x39); + x5 = _mm_shuffle_epi32(x5, 0x93); + x6 = _mm_shuffle_epi32(x6, 0x4e); + x7 = _mm_shuffle_epi32(x7, 0x39); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax - 1) & 0xff); + if (((byte)rax) != (0)) { + goto L_chacha20_sse3_last_crypt2_start; + } + x0 = _mm_add_epi32(x0, x10); + x1 = _mm_add_epi32(x1, x11); + x2 = _mm_add_epi32(x2, x12); + x3 = _mm_add_epi32(x3, x13); + x4 = _mm_add_epi32(x4, x10); + x5 = _mm_add_epi32(x5, x11); + x6 = _mm_add_epi32(x6, x12); + x7 = _mm_add_epi32(x7, x13); + x7 = _mm_add_epi32(x7, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + if (((word32)rcx) <= (0x40)) { + goto L_chacha20_sse3_last_lt64; + } + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_xor_si128(x0, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x1 = _mm_xor_si128(x1, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x1); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x2 = _mm_xor_si128(x2, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x2); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x3 = _mm_xor_si128(x3, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x3); + x0 = x4; + x1 = x5; + x2 = x6; + x3 = x7; + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 1); + rcx = (word32)((word32)rcx - 0x40); + rsi = (word64)(rsi + 0x40); + rdx = (word64)(rdx + 0x40); +L_chacha20_sse3_last_lt64: + r8 = (word64)(rdi + 80); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r8, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r8, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r8, 48), x3); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 1); + rax = (word32)((word32)rcx); + r9 = (word64)(0); + rax = (word32)((word32)rax & 7); + if (((word32)rax) == (0)) { + goto L_chacha20_sse3_last_start64; + } +L_chacha20_sse3_last_start8: + r10 = (word32)((word32)WC_L8(r8, r9)); + r10 = (r10 & ~(word64)0xff) | ((word64)(byte)((byte)r10 ^ WC_L8(rsi, + r9)) & 0xff); + WC_S8(rdx, r9) = (byte)((byte)r10); + r9 = (word32)((word32)r9 + 1); + zf1 = (word32)r9; + zf2 = (word32)rax; + if (((word32)r9) != ((word32)rax)) { + goto L_chacha20_sse3_last_start8; + } + if ((zf1) == (zf2)) { + goto L_chacha20_sse3_last_end64; + } +L_chacha20_sse3_last_start64: + r10 = (word64)(WC_L64(r8, r9)); + r10 = (word64)(r10 ^ WC_L64(rsi, r9)); + WC_S64(rdx, r9) = (word64)(r10); + r9 = (word32)((word32)r9 + 8); +L_chacha20_sse3_last_end64: + if (((word32)r9) != ((word32)rcx)) { + goto L_chacha20_sse3_last_start64; + } + rax = (word32)(0x40); + rax = (word32)((word32)rax - (word32)r9); + WC_S32(rdi, 76) = (word32)((word32)rax); +L_chacha20_sse3_last_done: + ; +} + +#endif /* HAVE_INTEL_SSSE3 */ +#ifdef HAVE_INTEL_AVX1 +XALIGNED(16) static const word64 L_chacha20_avx1_rotl8[] WC_X64I_UNUSED = { + 0x0605040702010003ULL, 0x0e0d0c0f0a09080bULL, +}; + +XALIGNED(16) static const word64 L_chacha20_avx1_rotl16[] WC_X64I_UNUSED = { + 0x0504070601000302ULL, 0x0d0c0f0e09080b0aULL, +}; + +XALIGNED(16) static const word64 L_chacha20_avx1_add[] WC_X64I_UNUSED = { + 0x0000000100000000ULL, 0x0000000300000002ULL, +}; + +XALIGNED(16) static const word64 L_chacha20_avx1_four[] WC_X64I_UNUSED = { + 0x0000000400000004ULL, 0x0000000400000004ULL, +}; + +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL void chacha_encrypt_avx1(ChaCha* ctx, const byte* m, byte* c, + word32 bytes) +{ + word64 rdi, rsi, rdx, rcx, rsp, r9, r10, r12, r13, r14, r15, rax, r8 = 0, + r11 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[52]; + word32 zf1; + word32 zf2; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rdx = (word64)(size_t)c; + rcx = (word64)(word32)bytes; + + rsp = (word64)(size_t)stk + 416; + rsp = (word64)(rsp - 400); + r9 = (word64)(rsp); + r10 = (word64)(rsp + 256); + r12 = (word64)((word64)(size_t)L_chacha20_avx1_rotl8); + r13 = (word64)((word64)(size_t)L_chacha20_avx1_rotl16); + r14 = (word64)((word64)(size_t)L_chacha20_avx1_add); + r15 = (word64)((word64)(size_t)L_chacha20_avx1_four); + r9 = (word64)(r9 + 0xf); + r10 = (word64)(r10 + 0xf); + r9 = (word64)(r9 & -16); + r10 = (word64)(r10 & -16); + rax = (word32)((word32)rcx); + rax = (word32)((word32)rax >> 8); + if (((word32)rax) == (0)) { + goto L_chacha20_avx1_end128; + } + x0 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)), 0); + x1 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 4)), 0); + x2 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 8)), 0); + x3 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 12)), 0); + x4 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)), 0); + x5 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 20)), 0); + x6 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 24)), 0); + x7 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 28)), 0); + x8 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)), 0); + x9 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 36)), 0); + x10 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 40)), 0); + x11 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 44)), 0); + x12 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)), 0); + x13 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 52)), 0); + x14 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 56)), 0); + x15 = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)WC_PR(rdi, 60)), 0); + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR(r14, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r9, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r9, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r9, 48), x3); + _mm_storeu_si128((__m128i*)WC_PW(r9, 64), x4); + _mm_storeu_si128((__m128i*)WC_PW(r9, 80), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 96), x6); + _mm_storeu_si128((__m128i*)WC_PW(r9, 112), x7); + _mm_storeu_si128((__m128i*)WC_PW(r9, 128), x8); + _mm_storeu_si128((__m128i*)WC_PW(r9, 144), x9); + _mm_storeu_si128((__m128i*)WC_PW(r9, 160), x10); + _mm_storeu_si128((__m128i*)WC_PW(r9, 176), x11); + _mm_storeu_si128((__m128i*)WC_PW(r9, 192), x12); + _mm_storeu_si128((__m128i*)WC_PW(r9, 208), x13); + _mm_storeu_si128((__m128i*)WC_PW(r9, 224), x14); + _mm_storeu_si128((__m128i*)WC_PW(r9, 240), x15); +L_chacha20_avx1_start128: + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_chacha20_avx1_loop128: + x0 = _mm_add_epi32(x0, x4); + x12 = _mm_xor_si128(x12, x0); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x12 = _mm_shuffle_epi8(x12, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x8 = _mm_add_epi32(x8, x12); + x4 = _mm_xor_si128(x4, x8); + x1 = _mm_add_epi32(x1, x5); + x13 = _mm_xor_si128(x13, x1); + x13 = _mm_shuffle_epi8(x13, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x9 = _mm_add_epi32(x9, x13); + x5 = _mm_xor_si128(x5, x9); + x2 = _mm_add_epi32(x2, x6); + x14 = _mm_xor_si128(x14, x2); + x14 = _mm_shuffle_epi8(x14, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x10 = _mm_add_epi32(x10, x14); + x6 = _mm_xor_si128(x6, x10); + x3 = _mm_add_epi32(x3, x7); + x15 = _mm_xor_si128(x15, x3); + x15 = _mm_shuffle_epi8(x15, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x11 = _mm_add_epi32(x11, x15); + x7 = _mm_xor_si128(x7, x11); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + x11 = _mm_srli_epi32(x4, 20); + x4 = _mm_slli_epi32(x4, 12); + x4 = _mm_xor_si128(x4, x11); + x11 = _mm_srli_epi32(x5, 20); + x5 = _mm_slli_epi32(x5, 12); + x5 = _mm_xor_si128(x5, x11); + x11 = _mm_srli_epi32(x6, 20); + x6 = _mm_slli_epi32(x6, 12); + x6 = _mm_xor_si128(x6, x11); + x11 = _mm_srli_epi32(x7, 20); + x7 = _mm_slli_epi32(x7, 12); + x7 = _mm_xor_si128(x7, x11); + x0 = _mm_add_epi32(x0, x4); + x12 = _mm_xor_si128(x12, x0); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x12 = _mm_shuffle_epi8(x12, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x8 = _mm_add_epi32(x8, x12); + x4 = _mm_xor_si128(x4, x8); + x1 = _mm_add_epi32(x1, x5); + x13 = _mm_xor_si128(x13, x1); + x13 = _mm_shuffle_epi8(x13, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x9 = _mm_add_epi32(x9, x13); + x5 = _mm_xor_si128(x5, x9); + x2 = _mm_add_epi32(x2, x6); + x14 = _mm_xor_si128(x14, x2); + x14 = _mm_shuffle_epi8(x14, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x10 = _mm_add_epi32(x10, x14); + x6 = _mm_xor_si128(x6, x10); + x3 = _mm_add_epi32(x3, x7); + x15 = _mm_xor_si128(x15, x3); + x15 = _mm_shuffle_epi8(x15, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x11 = _mm_add_epi32(x11, x15); + x7 = _mm_xor_si128(x7, x11); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + x11 = _mm_srli_epi32(x4, 25); + x4 = _mm_slli_epi32(x4, 7); + x4 = _mm_xor_si128(x4, x11); + x11 = _mm_srli_epi32(x5, 25); + x5 = _mm_slli_epi32(x5, 7); + x5 = _mm_xor_si128(x5, x11); + x11 = _mm_srli_epi32(x6, 25); + x6 = _mm_slli_epi32(x6, 7); + x6 = _mm_xor_si128(x6, x11); + x11 = _mm_srli_epi32(x7, 25); + x7 = _mm_slli_epi32(x7, 7); + x7 = _mm_xor_si128(x7, x11); + x0 = _mm_add_epi32(x0, x5); + x15 = _mm_xor_si128(x15, x0); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x15 = _mm_shuffle_epi8(x15, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x10 = _mm_add_epi32(x10, x15); + x5 = _mm_xor_si128(x5, x10); + x1 = _mm_add_epi32(x1, x6); + x12 = _mm_xor_si128(x12, x1); + x12 = _mm_shuffle_epi8(x12, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x11 = _mm_add_epi32(x11, x12); + x6 = _mm_xor_si128(x6, x11); + x2 = _mm_add_epi32(x2, x7); + x13 = _mm_xor_si128(x13, x2); + x13 = _mm_shuffle_epi8(x13, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x8 = _mm_add_epi32(x8, x13); + x7 = _mm_xor_si128(x7, x8); + x3 = _mm_add_epi32(x3, x4); + x14 = _mm_xor_si128(x14, x3); + x14 = _mm_shuffle_epi8(x14, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x9 = _mm_add_epi32(x9, x14); + x4 = _mm_xor_si128(x4, x9); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + x11 = _mm_srli_epi32(x5, 20); + x5 = _mm_slli_epi32(x5, 12); + x5 = _mm_xor_si128(x5, x11); + x11 = _mm_srli_epi32(x6, 20); + x6 = _mm_slli_epi32(x6, 12); + x6 = _mm_xor_si128(x6, x11); + x11 = _mm_srli_epi32(x7, 20); + x7 = _mm_slli_epi32(x7, 12); + x7 = _mm_xor_si128(x7, x11); + x11 = _mm_srli_epi32(x4, 20); + x4 = _mm_slli_epi32(x4, 12); + x4 = _mm_xor_si128(x4, x11); + x0 = _mm_add_epi32(x0, x5); + x15 = _mm_xor_si128(x15, x0); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x15 = _mm_shuffle_epi8(x15, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x10 = _mm_add_epi32(x10, x15); + x5 = _mm_xor_si128(x5, x10); + x1 = _mm_add_epi32(x1, x6); + x12 = _mm_xor_si128(x12, x1); + x12 = _mm_shuffle_epi8(x12, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x11 = _mm_add_epi32(x11, x12); + x6 = _mm_xor_si128(x6, x11); + x2 = _mm_add_epi32(x2, x7); + x13 = _mm_xor_si128(x13, x2); + x13 = _mm_shuffle_epi8(x13, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x8 = _mm_add_epi32(x8, x13); + x7 = _mm_xor_si128(x7, x8); + x3 = _mm_add_epi32(x3, x4); + x14 = _mm_xor_si128(x14, x3); + x14 = _mm_shuffle_epi8(x14, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x9 = _mm_add_epi32(x9, x14); + x4 = _mm_xor_si128(x4, x9); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + x11 = _mm_srli_epi32(x5, 25); + x5 = _mm_slli_epi32(x5, 7); + x5 = _mm_xor_si128(x5, x11); + x11 = _mm_srli_epi32(x6, 25); + x6 = _mm_slli_epi32(x6, 7); + x6 = _mm_xor_si128(x6, x11); + x11 = _mm_srli_epi32(x7, 25); + x7 = _mm_slli_epi32(x7, 7); + x7 = _mm_xor_si128(x7, x11); + x11 = _mm_srli_epi32(x4, 25); + x4 = _mm_slli_epi32(x4, 7); + x4 = _mm_xor_si128(x4, x11); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 - 1) & 0xff); + if (((byte)r8) != (0)) { + goto L_chacha20_avx1_loop128; + } + x11 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(r9, 16))); + x2 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(r9, 32))); + x3 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(r9, 48))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(r9, 64))); + x5 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(r9, 80))); + x6 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(r9, 96))); + x7 = _mm_add_epi32(x7, _mm_loadu_si128((const __m128i*)WC_PR(r9, 112))); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR(r9, 128))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(r9, 144))); + x10 = _mm_add_epi32(x10, _mm_loadu_si128((const __m128i*)WC_PR(r9, 160))); + x11 = _mm_add_epi32(x11, _mm_loadu_si128((const __m128i*)WC_PR(r9, 176))); + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR(r9, 192))); + x13 = _mm_add_epi32(x13, _mm_loadu_si128((const __m128i*)WC_PR(r9, 208))); + x14 = _mm_add_epi32(x14, _mm_loadu_si128((const __m128i*)WC_PR(r9, 224))); + x15 = _mm_add_epi32(x15, _mm_loadu_si128((const __m128i*)WC_PR(r9, 240))); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(r10, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(r10, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + _mm_storeu_si128((__m128i*)WC_PW(r10, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(r10, 80), x13); + _mm_storeu_si128((__m128i*)WC_PW(r10, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(r10, 112), x15); + x8 = _mm_unpacklo_epi32(x0, x1); + x9 = _mm_unpacklo_epi32(x2, x3); + x12 = _mm_unpackhi_epi32(x0, x1); + x13 = _mm_unpackhi_epi32(x2, x3); + x10 = _mm_unpacklo_epi32(x4, x5); + x11 = _mm_unpacklo_epi32(x6, x7); + x14 = _mm_unpackhi_epi32(x4, x5); + x15 = _mm_unpackhi_epi32(x6, x7); + x0 = _mm_unpacklo_epi64(x8, x9); + x1 = _mm_unpacklo_epi64(x10, x11); + x2 = _mm_unpackhi_epi64(x8, x9); + x3 = _mm_unpackhi_epi64(x10, x11); + x4 = _mm_unpacklo_epi64(x12, x13); + x5 = _mm_unpacklo_epi64(x14, x15); + x6 = _mm_unpackhi_epi64(x12, x13); + x7 = _mm_unpackhi_epi64(x14, x15); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x0 = _mm_xor_si128(x0, x8); + x1 = _mm_xor_si128(x1, x9); + x2 = _mm_xor_si128(x2, x10); + x3 = _mm_xor_si128(x3, x11); + x4 = _mm_xor_si128(x4, x12); + x5 = _mm_xor_si128(x5, x13); + x6 = _mm_xor_si128(x6, x14); + x7 = _mm_xor_si128(x7, x15); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 128), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 144), x5); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 192), x6); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 208), x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_unpacklo_epi32(x0, x1); + x9 = _mm_unpacklo_epi32(x2, x3); + x12 = _mm_unpackhi_epi32(x0, x1); + x13 = _mm_unpackhi_epi32(x2, x3); + x10 = _mm_unpacklo_epi32(x4, x5); + x11 = _mm_unpacklo_epi32(x6, x7); + x14 = _mm_unpackhi_epi32(x4, x5); + x15 = _mm_unpackhi_epi32(x6, x7); + x0 = _mm_unpacklo_epi64(x8, x9); + x1 = _mm_unpacklo_epi64(x10, x11); + x2 = _mm_unpackhi_epi64(x8, x9); + x3 = _mm_unpackhi_epi64(x10, x11); + x4 = _mm_unpacklo_epi64(x12, x13); + x5 = _mm_unpacklo_epi64(x14, x15); + x6 = _mm_unpackhi_epi64(x12, x13); + x7 = _mm_unpackhi_epi64(x14, x15); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 240)); + x0 = _mm_xor_si128(x0, x8); + x1 = _mm_xor_si128(x1, x9); + x2 = _mm_xor_si128(x2, x10); + x3 = _mm_xor_si128(x3, x11); + x4 = _mm_xor_si128(x4, x12); + x5 = _mm_xor_si128(x5, x13); + x6 = _mm_xor_si128(x6, x14); + x7 = _mm_xor_si128(x7, x15); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 160), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 176), x5); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 224), x6); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 240), x7); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 192)); + rsi = (word64)(rsi + 0x100); + rdx = (word64)(rdx + 0x100); + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR(r15, 0))); + rcx = (word32)((word32)rcx - 0x100); + _mm_storeu_si128((__m128i*)WC_PW(r9, 192), x12); + if (((word32)rcx) < (0x100)) { + goto L_chacha20_avx1_done128; + } + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 48)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 80)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 112)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 128)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 144)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 160)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 176)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 192)); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 208)); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 224)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 240)); + goto L_chacha20_avx1_start128; +L_chacha20_avx1_done128: + rax = (word32)((word32)rax << 2); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + (word32)rax); +L_chacha20_avx1_end128: + if (((word32)rcx) < (0x40)) { + goto L_chacha20_avx1_block_done; + } +L_chacha20_avx1_block_start: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x5 = x0; + x6 = x1; + x7 = x2; + x8 = x3; + rax = (rax & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_chacha20_avx1_block_crypt_start: + x0 = _mm_add_epi32(x0, x1); + x3 = _mm_xor_si128(x3, x0); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x2 = _mm_add_epi32(x2, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_srli_epi32(x1, 20); + x1 = _mm_slli_epi32(x1, 12); + x1 = _mm_xor_si128(x1, x4); + x0 = _mm_add_epi32(x0, x1); + x3 = _mm_xor_si128(x3, x0); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x2 = _mm_add_epi32(x2, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_srli_epi32(x1, 25); + x1 = _mm_slli_epi32(x1, 7); + x1 = _mm_xor_si128(x1, x4); + x1 = _mm_shuffle_epi32(x1, 0x39); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x3 = _mm_shuffle_epi32(x3, 0x93); + x0 = _mm_add_epi32(x0, x1); + x3 = _mm_xor_si128(x3, x0); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x2 = _mm_add_epi32(x2, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_srli_epi32(x1, 20); + x1 = _mm_slli_epi32(x1, 12); + x1 = _mm_xor_si128(x1, x4); + x0 = _mm_add_epi32(x0, x1); + x3 = _mm_xor_si128(x3, x0); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x2 = _mm_add_epi32(x2, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_srli_epi32(x1, 25); + x1 = _mm_slli_epi32(x1, 7); + x1 = _mm_xor_si128(x1, x4); + x1 = _mm_shuffle_epi32(x1, 0x93); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x3 = _mm_shuffle_epi32(x3, 0x39); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax - 1) & 0xff); + if (((byte)rax) != (0)) { + goto L_chacha20_avx1_block_crypt_start; + } + x0 = _mm_add_epi32(x0, x5); + x1 = _mm_add_epi32(x1, x6); + x2 = _mm_add_epi32(x2, x7); + x3 = _mm_add_epi32(x3, x8); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_xor_si128(x0, x5); + x1 = _mm_xor_si128(x1, x6); + x2 = _mm_xor_si128(x2, x7); + x3 = _mm_xor_si128(x3, x8); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x3); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 1); + rcx = (word32)((word32)rcx - 0x40); + rsi = (word64)(rsi + 0x40); + rdx = (word64)(rdx + 0x40); + if (((word32)rcx) >= (0x40)) { + goto L_chacha20_avx1_block_start; + } +L_chacha20_avx1_block_done: + if (((word32)rcx) == (0)) { + goto L_chacha20_avx1_partial_done; + } + r10 = (word64)(rdi + 80); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rdi, 48)); + x5 = x0; + x6 = x1; + x7 = x2; + x8 = x3; + rax = (rax & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_chacha20_avx1_partial_crypt_start: + x0 = _mm_add_epi32(x0, x1); + x3 = _mm_xor_si128(x3, x0); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x2 = _mm_add_epi32(x2, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_srli_epi32(x1, 20); + x1 = _mm_slli_epi32(x1, 12); + x1 = _mm_xor_si128(x1, x4); + x0 = _mm_add_epi32(x0, x1); + x3 = _mm_xor_si128(x3, x0); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x2 = _mm_add_epi32(x2, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_srli_epi32(x1, 25); + x1 = _mm_slli_epi32(x1, 7); + x1 = _mm_xor_si128(x1, x4); + x1 = _mm_shuffle_epi32(x1, 0x39); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x3 = _mm_shuffle_epi32(x3, 0x93); + x0 = _mm_add_epi32(x0, x1); + x3 = _mm_xor_si128(x3, x0); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + x2 = _mm_add_epi32(x2, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_srli_epi32(x1, 20); + x1 = _mm_slli_epi32(x1, 12); + x1 = _mm_xor_si128(x1, x4); + x0 = _mm_add_epi32(x0, x1); + x3 = _mm_xor_si128(x3, x0); + x3 = _mm_shuffle_epi8(x3, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + x2 = _mm_add_epi32(x2, x3); + x1 = _mm_xor_si128(x1, x2); + x4 = _mm_srli_epi32(x1, 25); + x1 = _mm_slli_epi32(x1, 7); + x1 = _mm_xor_si128(x1, x4); + x1 = _mm_shuffle_epi32(x1, 0x93); + x2 = _mm_shuffle_epi32(x2, 0x4e); + x3 = _mm_shuffle_epi32(x3, 0x39); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax - 1) & 0xff); + if (((byte)rax) != (0)) { + goto L_chacha20_avx1_partial_crypt_start; + } + x0 = _mm_add_epi32(x0, x5); + x1 = _mm_add_epi32(x1, x6); + x2 = _mm_add_epi32(x2, x7); + x3 = _mm_add_epi32(x3, x8); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r10, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r10, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x3); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 1); + r8 = (word32)((word32)rcx); + r11 = (word64)(0); + r8 = (word32)((word32)r8 & 7); + if (((word32)r8) == (0)) { + goto L_chacha20_avx1_partial_start64; + } +L_chacha20_avx1_partial_start8: + rax = (word32)((word32)WC_L8(r10, r11)); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax ^ WC_L8(rsi, + r11)) & 0xff); + WC_S8(rdx, r11) = (byte)((byte)rax); + r11 = (word32)((word32)r11 + 1); + zf1 = (word32)r11; + zf2 = (word32)r8; + if (((word32)r11) != ((word32)r8)) { + goto L_chacha20_avx1_partial_start8; + } + if ((zf1) == (zf2)) { + goto L_chacha20_avx1_partial_end64; + } +L_chacha20_avx1_partial_start64: + rax = (word64)(WC_L64(r10, r11)); + rax = (word64)(rax ^ WC_L64(rsi, r11)); + WC_S64(rdx, r11) = (word64)(rax); + r11 = (word32)((word32)r11 + 8); +L_chacha20_avx1_partial_end64: + if (((word32)r11) != ((word32)rcx)) { + goto L_chacha20_avx1_partial_start64; + } + r8 = (word32)(0x40); + r8 = (word32)((word32)r8 - (word32)r11); + WC_S32(rdi, 76) = (word32)((word32)r8); +L_chacha20_avx1_partial_done: + ; +} + +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +XALIGNED(32) static const word64 L_chacha20_avx2_rotl8[] WC_X64I_UNUSED = { + 0x0605040702010003ULL, 0x0e0d0c0f0a09080bULL, + 0x0605040702010003ULL, 0x0e0d0c0f0a09080bULL, +}; + +XALIGNED(32) static const word64 L_chacha20_avx2_rotl16[] WC_X64I_UNUSED = { + 0x0504070601000302ULL, 0x0d0c0f0e09080b0aULL, + 0x0504070601000302ULL, 0x0d0c0f0e09080b0aULL, +}; + +XALIGNED(32) static const word64 L_chacha20_avx2_add[] WC_X64I_UNUSED = { + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000500000004ULL, 0x0000000700000006ULL, +}; + +XALIGNED(32) static const word64 L_chacha20_avx2_eight[] WC_X64I_UNUSED = { + 0x0000000800000008ULL, 0x0000000800000008ULL, + 0x0000000800000008ULL, 0x0000000800000008ULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void chacha_encrypt_avx2(ChaCha* ctx, const byte* m, byte* c, + word32 bytes) +{ + word64 rdi, rsi, rdx, rcx, rsp, r9, r11, r12, r13, r14, r10, rax, r8 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[100]; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rdx = (word64)(size_t)c; + rcx = (word64)(word32)bytes; + + rsp = (word64)(size_t)stk + 800; + rsp = (word64)(rsp - 800); + r9 = (word64)(rsp); + r11 = (word64)((word64)(size_t)L_chacha20_avx2_rotl8); + r12 = (word64)((word64)(size_t)L_chacha20_avx2_rotl16); + r13 = (word64)((word64)(size_t)L_chacha20_avx2_add); + r14 = (word64)((word64)(size_t)L_chacha20_avx2_eight); + r10 = (word64)(rsp + 512); + r9 = (word64)(r9 + 0x1f); + r10 = (word64)(r10 + 0x1f); + r9 = (word64)(r9 & -32); + r10 = (word64)(r10 & -32); + rax = (word32)((word32)rcx); + rax = (word32)((word32)rax >> 9); + if (((word32)rax) == (0)) { + goto L_chacha20_avx2_end256; + } + y0 = _mm256_set1_epi32((int)WC_L32(rdi, 0)); + y1 = _mm256_set1_epi32((int)WC_L32(rdi, 4)); + y2 = _mm256_set1_epi32((int)WC_L32(rdi, 8)); + y3 = _mm256_set1_epi32((int)WC_L32(rdi, 12)); + y4 = _mm256_set1_epi32((int)WC_L32(rdi, 16)); + y5 = _mm256_set1_epi32((int)WC_L32(rdi, 20)); + y6 = _mm256_set1_epi32((int)WC_L32(rdi, 24)); + y7 = _mm256_set1_epi32((int)WC_L32(rdi, 28)); + y8 = _mm256_set1_epi32((int)WC_L32(rdi, 32)); + y9 = _mm256_set1_epi32((int)WC_L32(rdi, 36)); + y10 = _mm256_set1_epi32((int)WC_L32(rdi, 40)); + y11 = _mm256_set1_epi32((int)WC_L32(rdi, 44)); + y12 = _mm256_set1_epi32((int)WC_L32(rdi, 48)); + y13 = _mm256_set1_epi32((int)WC_L32(rdi, 52)); + y14 = _mm256_set1_epi32((int)WC_L32(rdi, 56)); + y15 = _mm256_set1_epi32((int)WC_L32(rdi, 60)); + y12 = _mm256_add_epi32(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 224), y7); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 256), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 288), y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 320), y10); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 352), y11); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 384), y12); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 416), y13); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 448), y14); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 480), y15); +L_chacha20_avx2_start256: + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y11); +L_chacha20_avx2_loop256: + y0 = _mm256_add_epi32(y0, y4); + y12 = _mm256_xor_si256(y12, y0); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y12 = _mm256_shuffle_epi8(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y8 = _mm256_add_epi32(y8, y12); + y4 = _mm256_xor_si256(y4, y8); + y1 = _mm256_add_epi32(y1, y5); + y13 = _mm256_xor_si256(y13, y1); + y13 = _mm256_shuffle_epi8(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y9 = _mm256_add_epi32(y9, y13); + y5 = _mm256_xor_si256(y5, y9); + y2 = _mm256_add_epi32(y2, y6); + y14 = _mm256_xor_si256(y14, y2); + y14 = _mm256_shuffle_epi8(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y10 = _mm256_add_epi32(y10, y14); + y6 = _mm256_xor_si256(y6, y10); + y3 = _mm256_add_epi32(y3, y7); + y15 = _mm256_xor_si256(y15, y3); + y15 = _mm256_shuffle_epi8(y15, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y11 = _mm256_add_epi32(y11, y15); + y7 = _mm256_xor_si256(y7, y11); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y11); + y11 = _mm256_srli_epi32(y4, 20); + y4 = _mm256_slli_epi32(y4, 12); + y4 = _mm256_xor_si256(y4, y11); + y11 = _mm256_srli_epi32(y5, 20); + y5 = _mm256_slli_epi32(y5, 12); + y5 = _mm256_xor_si256(y5, y11); + y11 = _mm256_srli_epi32(y6, 20); + y6 = _mm256_slli_epi32(y6, 12); + y6 = _mm256_xor_si256(y6, y11); + y11 = _mm256_srli_epi32(y7, 20); + y7 = _mm256_slli_epi32(y7, 12); + y7 = _mm256_xor_si256(y7, y11); + y0 = _mm256_add_epi32(y0, y4); + y12 = _mm256_xor_si256(y12, y0); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y12 = _mm256_shuffle_epi8(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y8 = _mm256_add_epi32(y8, y12); + y4 = _mm256_xor_si256(y4, y8); + y1 = _mm256_add_epi32(y1, y5); + y13 = _mm256_xor_si256(y13, y1); + y13 = _mm256_shuffle_epi8(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y9 = _mm256_add_epi32(y9, y13); + y5 = _mm256_xor_si256(y5, y9); + y2 = _mm256_add_epi32(y2, y6); + y14 = _mm256_xor_si256(y14, y2); + y14 = _mm256_shuffle_epi8(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y10 = _mm256_add_epi32(y10, y14); + y6 = _mm256_xor_si256(y6, y10); + y3 = _mm256_add_epi32(y3, y7); + y15 = _mm256_xor_si256(y15, y3); + y15 = _mm256_shuffle_epi8(y15, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y11 = _mm256_add_epi32(y11, y15); + y7 = _mm256_xor_si256(y7, y11); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y11); + y11 = _mm256_srli_epi32(y4, 25); + y4 = _mm256_slli_epi32(y4, 7); + y4 = _mm256_xor_si256(y4, y11); + y11 = _mm256_srli_epi32(y5, 25); + y5 = _mm256_slli_epi32(y5, 7); + y5 = _mm256_xor_si256(y5, y11); + y11 = _mm256_srli_epi32(y6, 25); + y6 = _mm256_slli_epi32(y6, 7); + y6 = _mm256_xor_si256(y6, y11); + y11 = _mm256_srli_epi32(y7, 25); + y7 = _mm256_slli_epi32(y7, 7); + y7 = _mm256_xor_si256(y7, y11); + y0 = _mm256_add_epi32(y0, y5); + y15 = _mm256_xor_si256(y15, y0); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y15 = _mm256_shuffle_epi8(y15, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y10 = _mm256_add_epi32(y10, y15); + y5 = _mm256_xor_si256(y5, y10); + y1 = _mm256_add_epi32(y1, y6); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_shuffle_epi8(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y11 = _mm256_add_epi32(y11, y12); + y6 = _mm256_xor_si256(y6, y11); + y2 = _mm256_add_epi32(y2, y7); + y13 = _mm256_xor_si256(y13, y2); + y13 = _mm256_shuffle_epi8(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y8 = _mm256_add_epi32(y8, y13); + y7 = _mm256_xor_si256(y7, y8); + y3 = _mm256_add_epi32(y3, y4); + y14 = _mm256_xor_si256(y14, y3); + y14 = _mm256_shuffle_epi8(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y9 = _mm256_add_epi32(y9, y14); + y4 = _mm256_xor_si256(y4, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y11); + y11 = _mm256_srli_epi32(y5, 20); + y5 = _mm256_slli_epi32(y5, 12); + y5 = _mm256_xor_si256(y5, y11); + y11 = _mm256_srli_epi32(y6, 20); + y6 = _mm256_slli_epi32(y6, 12); + y6 = _mm256_xor_si256(y6, y11); + y11 = _mm256_srli_epi32(y7, 20); + y7 = _mm256_slli_epi32(y7, 12); + y7 = _mm256_xor_si256(y7, y11); + y11 = _mm256_srli_epi32(y4, 20); + y4 = _mm256_slli_epi32(y4, 12); + y4 = _mm256_xor_si256(y4, y11); + y0 = _mm256_add_epi32(y0, y5); + y15 = _mm256_xor_si256(y15, y0); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y15 = _mm256_shuffle_epi8(y15, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y10 = _mm256_add_epi32(y10, y15); + y5 = _mm256_xor_si256(y5, y10); + y1 = _mm256_add_epi32(y1, y6); + y12 = _mm256_xor_si256(y12, y1); + y12 = _mm256_shuffle_epi8(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y11 = _mm256_add_epi32(y11, y12); + y6 = _mm256_xor_si256(y6, y11); + y2 = _mm256_add_epi32(y2, y7); + y13 = _mm256_xor_si256(y13, y2); + y13 = _mm256_shuffle_epi8(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y8 = _mm256_add_epi32(y8, y13); + y7 = _mm256_xor_si256(y7, y8); + y3 = _mm256_add_epi32(y3, y4); + y14 = _mm256_xor_si256(y14, y3); + y14 = _mm256_shuffle_epi8(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y9 = _mm256_add_epi32(y9, y14); + y4 = _mm256_xor_si256(y4, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y11); + y11 = _mm256_srli_epi32(y5, 25); + y5 = _mm256_slli_epi32(y5, 7); + y5 = _mm256_xor_si256(y5, y11); + y11 = _mm256_srli_epi32(y6, 25); + y6 = _mm256_slli_epi32(y6, 7); + y6 = _mm256_xor_si256(y6, y11); + y11 = _mm256_srli_epi32(y7, 25); + y7 = _mm256_slli_epi32(y7, 7); + y7 = _mm256_xor_si256(y7, y11); + y11 = _mm256_srli_epi32(y4, 25); + y4 = _mm256_slli_epi32(y4, 7); + y4 = _mm256_xor_si256(y4, y11); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 - 1) & 0xff); + if (((byte)r8) != (0)) { + goto L_chacha20_avx2_loop256; + } + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y0 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(r9, 0))); + y1 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 32))); + y2 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 64))); + y3 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 96))); + y4 = _mm256_add_epi32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 128))); + y5 = _mm256_add_epi32(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 160))); + y6 = _mm256_add_epi32(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 192))); + y7 = _mm256_add_epi32(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 224))); + y8 = _mm256_add_epi32(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 256))); + y9 = _mm256_add_epi32(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 288))); + y10 = _mm256_add_epi32(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 320))); + y11 = _mm256_add_epi32(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 352))); + y12 = _mm256_add_epi32(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 384))); + y13 = _mm256_add_epi32(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 416))); + y14 = _mm256_add_epi32(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 448))); + y15 = _mm256_add_epi32(y15, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 480))); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), y10); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y11); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), y12); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), y13); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), y14); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), y15); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpacklo_epi32(y2, y3); + y12 = _mm256_unpackhi_epi32(y0, y1); + y13 = _mm256_unpackhi_epi32(y2, y3); + y10 = _mm256_unpacklo_epi32(y4, y5); + y11 = _mm256_unpacklo_epi32(y6, y7); + y14 = _mm256_unpackhi_epi32(y4, y5); + y15 = _mm256_unpackhi_epi32(y6, y7); + y0 = _mm256_unpacklo_epi64(y8, y9); + y1 = _mm256_unpacklo_epi64(y10, y11); + y2 = _mm256_unpackhi_epi64(y8, y9); + y3 = _mm256_unpackhi_epi64(y10, y11); + y4 = _mm256_unpacklo_epi64(y12, y13); + y5 = _mm256_unpacklo_epi64(y14, y15); + y6 = _mm256_unpackhi_epi64(y12, y13); + y7 = _mm256_unpackhi_epi64(y14, y15); + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y12 = _mm256_permute2x128_si256(y0, y1, 0x31); + y13 = _mm256_permute2x128_si256(y2, y3, 0x31); + y10 = _mm256_permute2x128_si256(y4, y5, 0x20); + y11 = _mm256_permute2x128_si256(y6, y7, 0x20); + y14 = _mm256_permute2x128_si256(y4, y5, 0x31); + y15 = _mm256_permute2x128_si256(y6, y7, 0x31); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y8 = _mm256_xor_si256(y8, y0); + y9 = _mm256_xor_si256(y9, y1); + y10 = _mm256_xor_si256(y10, y2); + y11 = _mm256_xor_si256(y11, y3); + y12 = _mm256_xor_si256(y12, y4); + y13 = _mm256_xor_si256(y13, y5); + y14 = _mm256_xor_si256(y14, y6); + y15 = _mm256_xor_si256(y15, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), y15); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpacklo_epi32(y2, y3); + y12 = _mm256_unpackhi_epi32(y0, y1); + y13 = _mm256_unpackhi_epi32(y2, y3); + y10 = _mm256_unpacklo_epi32(y4, y5); + y11 = _mm256_unpacklo_epi32(y6, y7); + y14 = _mm256_unpackhi_epi32(y4, y5); + y15 = _mm256_unpackhi_epi32(y6, y7); + y0 = _mm256_unpacklo_epi64(y8, y9); + y1 = _mm256_unpacklo_epi64(y10, y11); + y2 = _mm256_unpackhi_epi64(y8, y9); + y3 = _mm256_unpackhi_epi64(y10, y11); + y4 = _mm256_unpacklo_epi64(y12, y13); + y5 = _mm256_unpacklo_epi64(y14, y15); + y6 = _mm256_unpackhi_epi64(y12, y13); + y7 = _mm256_unpackhi_epi64(y14, y15); + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y12 = _mm256_permute2x128_si256(y0, y1, 0x31); + y13 = _mm256_permute2x128_si256(y2, y3, 0x31); + y10 = _mm256_permute2x128_si256(y4, y5, 0x20); + y11 = _mm256_permute2x128_si256(y6, y7, 0x20); + y14 = _mm256_permute2x128_si256(y4, y5, 0x31); + y15 = _mm256_permute2x128_si256(y6, y7, 0x31); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y8 = _mm256_xor_si256(y8, y0); + y9 = _mm256_xor_si256(y9, y1); + y10 = _mm256_xor_si256(y10, y2); + y11 = _mm256_xor_si256(y11, y3); + y12 = _mm256_xor_si256(y12, y4); + y13 = _mm256_xor_si256(y13, y5); + y14 = _mm256_xor_si256(y14, y6); + y15 = _mm256_xor_si256(y15, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), y15); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 384)); + rsi = (word64)(rsi + 0x200); + rdx = (word64)(rdx + 0x200); + y12 = _mm256_add_epi32(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 0))); + rcx = (word32)((word32)rcx - 0x200); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 384), y12); + if (((word32)rcx) < (0x200)) { + goto L_chacha20_avx2_done256; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 256)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 288)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 352)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 384)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 416)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 448)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 480)); + goto L_chacha20_avx2_start256; +L_chacha20_avx2_done256: + rax = (word32)((word32)rax << 3); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + (word32)rax); +L_chacha20_avx2_end256: + (void)chacha_encrypt_avx1((ChaCha*)(size_t)rdi, (const byte*)(size_t)rsi, ( + byte*)(size_t)rdx, (word32)rcx); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX512 +XALIGNED(16) static const word64 L_chacha20_avx512vl_add[] WC_X64I_UNUSED = { + 0x0000000100000000ULL, 0x0000000300000002ULL, +}; + +XALIGNED(16) static const word64 L_chacha20_avx512vl_four[] WC_X64I_UNUSED = { + 0x0000000400000004ULL, 0x0000000400000004ULL, +}; + +WC_X64I_TARGET("avx512f,avx512vl") +WOLFSSL_LOCAL void chacha_encrypt_avx512vl(ChaCha* ctx, const byte* m, byte* c, + word32 bytes) +{ + word64 rdi, rsi, rdx, rcx, rsp, r9, r10, r12, r13, rax, r8 = 0, r11 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), + x14 = _mm_setzero_si128(), x15 = _mm_setzero_si128(); + XALIGNED(32) WC_X64I_SLOT stk[52]; + word32 zf1; + word32 zf2; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rdx = (word64)(size_t)c; + rcx = (word64)(word32)bytes; + + rsp = (word64)(size_t)stk + 416; + rsp = (word64)(rsp - 400); + r9 = (word64)(rsp); + r10 = (word64)(rsp + 256); + r12 = (word64)((word64)(size_t)L_chacha20_avx512vl_add); + r13 = (word64)((word64)(size_t)L_chacha20_avx512vl_four); + r9 = (word64)(r9 + 0xf); + r10 = (word64)(r10 + 0xf); + r9 = (word64)(r9 & -16); + r10 = (word64)(r10 & -16); + rax = (word32)((word32)rcx); + rax = (word32)((word32)rax >> 8); + if (((word32)rax) == (0)) { + goto L_chacha20_avx512vl_end128; + } + x0 = _mm_set1_epi32((int)WC_L32(rdi, 0)); + x1 = _mm_set1_epi32((int)WC_L32(rdi, 4)); + x2 = _mm_set1_epi32((int)WC_L32(rdi, 8)); + x3 = _mm_set1_epi32((int)WC_L32(rdi, 12)); + x4 = _mm_set1_epi32((int)WC_L32(rdi, 16)); + x5 = _mm_set1_epi32((int)WC_L32(rdi, 20)); + x6 = _mm_set1_epi32((int)WC_L32(rdi, 24)); + x7 = _mm_set1_epi32((int)WC_L32(rdi, 28)); + x8 = _mm_set1_epi32((int)WC_L32(rdi, 32)); + x9 = _mm_set1_epi32((int)WC_L32(rdi, 36)); + x10 = _mm_set1_epi32((int)WC_L32(rdi, 40)); + x11 = _mm_set1_epi32((int)WC_L32(rdi, 44)); + x12 = _mm_set1_epi32((int)WC_L32(rdi, 48)); + x13 = _mm_set1_epi32((int)WC_L32(rdi, 52)); + x14 = _mm_set1_epi32((int)WC_L32(rdi, 56)); + x15 = _mm_set1_epi32((int)WC_L32(rdi, 60)); + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r9, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r9, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r9, 48), x3); + _mm_storeu_si128((__m128i*)WC_PW(r9, 64), x4); + _mm_storeu_si128((__m128i*)WC_PW(r9, 80), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 96), x6); + _mm_storeu_si128((__m128i*)WC_PW(r9, 112), x7); + _mm_storeu_si128((__m128i*)WC_PW(r9, 128), x8); + _mm_storeu_si128((__m128i*)WC_PW(r9, 144), x9); + _mm_storeu_si128((__m128i*)WC_PW(r9, 160), x10); + _mm_storeu_si128((__m128i*)WC_PW(r9, 176), x11); + _mm_storeu_si128((__m128i*)WC_PW(r9, 192), x12); + _mm_storeu_si128((__m128i*)WC_PW(r9, 208), x13); + _mm_storeu_si128((__m128i*)WC_PW(r9, 224), x14); + _mm_storeu_si128((__m128i*)WC_PW(r9, 240), x15); +L_chacha20_avx512vl_start128: + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_chacha20_avx512vl_loop128: + x0 = _mm_add_epi32(x0, x4); + x1 = _mm_add_epi32(x1, x5); + x2 = _mm_add_epi32(x2, x6); + x3 = _mm_add_epi32(x3, x7); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + x12 = _mm_rol_epi32(x12, 16); + x13 = _mm_rol_epi32(x13, 16); + x14 = _mm_rol_epi32(x14, 16); + x15 = _mm_rol_epi32(x15, 16); + x8 = _mm_add_epi32(x8, x12); + x9 = _mm_add_epi32(x9, x13); + x10 = _mm_add_epi32(x10, x14); + x11 = _mm_add_epi32(x11, x15); + x4 = _mm_xor_si128(x4, x8); + x5 = _mm_xor_si128(x5, x9); + x6 = _mm_xor_si128(x6, x10); + x7 = _mm_xor_si128(x7, x11); + x4 = _mm_rol_epi32(x4, 12); + x5 = _mm_rol_epi32(x5, 12); + x6 = _mm_rol_epi32(x6, 12); + x7 = _mm_rol_epi32(x7, 12); + x0 = _mm_add_epi32(x0, x4); + x1 = _mm_add_epi32(x1, x5); + x2 = _mm_add_epi32(x2, x6); + x3 = _mm_add_epi32(x3, x7); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + x12 = _mm_rol_epi32(x12, 8); + x13 = _mm_rol_epi32(x13, 8); + x14 = _mm_rol_epi32(x14, 8); + x15 = _mm_rol_epi32(x15, 8); + x8 = _mm_add_epi32(x8, x12); + x9 = _mm_add_epi32(x9, x13); + x10 = _mm_add_epi32(x10, x14); + x11 = _mm_add_epi32(x11, x15); + x4 = _mm_xor_si128(x4, x8); + x5 = _mm_xor_si128(x5, x9); + x6 = _mm_xor_si128(x6, x10); + x7 = _mm_xor_si128(x7, x11); + x4 = _mm_rol_epi32(x4, 7); + x5 = _mm_rol_epi32(x5, 7); + x6 = _mm_rol_epi32(x6, 7); + x7 = _mm_rol_epi32(x7, 7); + x0 = _mm_add_epi32(x0, x5); + x1 = _mm_add_epi32(x1, x6); + x2 = _mm_add_epi32(x2, x7); + x3 = _mm_add_epi32(x3, x4); + x15 = _mm_xor_si128(x15, x0); + x12 = _mm_xor_si128(x12, x1); + x13 = _mm_xor_si128(x13, x2); + x14 = _mm_xor_si128(x14, x3); + x15 = _mm_rol_epi32(x15, 16); + x12 = _mm_rol_epi32(x12, 16); + x13 = _mm_rol_epi32(x13, 16); + x14 = _mm_rol_epi32(x14, 16); + x10 = _mm_add_epi32(x10, x15); + x11 = _mm_add_epi32(x11, x12); + x8 = _mm_add_epi32(x8, x13); + x9 = _mm_add_epi32(x9, x14); + x5 = _mm_xor_si128(x5, x10); + x6 = _mm_xor_si128(x6, x11); + x7 = _mm_xor_si128(x7, x8); + x4 = _mm_xor_si128(x4, x9); + x5 = _mm_rol_epi32(x5, 12); + x6 = _mm_rol_epi32(x6, 12); + x7 = _mm_rol_epi32(x7, 12); + x4 = _mm_rol_epi32(x4, 12); + x0 = _mm_add_epi32(x0, x5); + x1 = _mm_add_epi32(x1, x6); + x2 = _mm_add_epi32(x2, x7); + x3 = _mm_add_epi32(x3, x4); + x15 = _mm_xor_si128(x15, x0); + x12 = _mm_xor_si128(x12, x1); + x13 = _mm_xor_si128(x13, x2); + x14 = _mm_xor_si128(x14, x3); + x15 = _mm_rol_epi32(x15, 8); + x12 = _mm_rol_epi32(x12, 8); + x13 = _mm_rol_epi32(x13, 8); + x14 = _mm_rol_epi32(x14, 8); + x10 = _mm_add_epi32(x10, x15); + x11 = _mm_add_epi32(x11, x12); + x8 = _mm_add_epi32(x8, x13); + x9 = _mm_add_epi32(x9, x14); + x5 = _mm_xor_si128(x5, x10); + x6 = _mm_xor_si128(x6, x11); + x7 = _mm_xor_si128(x7, x8); + x4 = _mm_xor_si128(x4, x9); + x5 = _mm_rol_epi32(x5, 7); + x6 = _mm_rol_epi32(x6, 7); + x7 = _mm_rol_epi32(x7, 7); + x4 = _mm_rol_epi32(x4, 7); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 - 1) & 0xff); + if (((byte)r8) != (0)) { + goto L_chacha20_avx512vl_loop128; + } + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(r9, 16))); + x2 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(r9, 32))); + x3 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(r9, 48))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(r9, 64))); + x5 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(r9, 80))); + x6 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(r9, 96))); + x7 = _mm_add_epi32(x7, _mm_loadu_si128((const __m128i*)WC_PR(r9, 112))); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR(r9, 128))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(r9, 144))); + x10 = _mm_add_epi32(x10, _mm_loadu_si128((const __m128i*)WC_PR(r9, 160))); + x11 = _mm_add_epi32(x11, _mm_loadu_si128((const __m128i*)WC_PR(r9, 176))); + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR(r9, 192))); + x13 = _mm_add_epi32(x13, _mm_loadu_si128((const __m128i*)WC_PR(r9, 208))); + x14 = _mm_add_epi32(x14, _mm_loadu_si128((const __m128i*)WC_PR(r9, 224))); + x15 = _mm_add_epi32(x15, _mm_loadu_si128((const __m128i*)WC_PR(r9, 240))); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(r10, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(r10, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + _mm_storeu_si128((__m128i*)WC_PW(r10, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(r10, 80), x13); + _mm_storeu_si128((__m128i*)WC_PW(r10, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(r10, 112), x15); + x8 = _mm_unpacklo_epi32(x0, x1); + x9 = _mm_unpacklo_epi32(x2, x3); + x12 = _mm_unpackhi_epi32(x0, x1); + x13 = _mm_unpackhi_epi32(x2, x3); + x10 = _mm_unpacklo_epi32(x4, x5); + x11 = _mm_unpacklo_epi32(x6, x7); + x14 = _mm_unpackhi_epi32(x4, x5); + x15 = _mm_unpackhi_epi32(x6, x7); + x0 = _mm_unpacklo_epi64(x8, x9); + x1 = _mm_unpacklo_epi64(x10, x11); + x2 = _mm_unpackhi_epi64(x8, x9); + x3 = _mm_unpackhi_epi64(x10, x11); + x4 = _mm_unpacklo_epi64(x12, x13); + x5 = _mm_unpacklo_epi64(x14, x15); + x6 = _mm_unpackhi_epi64(x12, x13); + x7 = _mm_unpackhi_epi64(x14, x15); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x0 = _mm_xor_si128(x0, x8); + x1 = _mm_xor_si128(x1, x9); + x2 = _mm_xor_si128(x2, x10); + x3 = _mm_xor_si128(x3, x11); + x4 = _mm_xor_si128(x4, x12); + x5 = _mm_xor_si128(x5, x13); + x6 = _mm_xor_si128(x6, x14); + x7 = _mm_xor_si128(x7, x15); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 64), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 80), x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 128), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 144), x5); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 192), x6); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 208), x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_unpacklo_epi32(x0, x1); + x9 = _mm_unpacklo_epi32(x2, x3); + x12 = _mm_unpackhi_epi32(x0, x1); + x13 = _mm_unpackhi_epi32(x2, x3); + x10 = _mm_unpacklo_epi32(x4, x5); + x11 = _mm_unpacklo_epi32(x6, x7); + x14 = _mm_unpackhi_epi32(x4, x5); + x15 = _mm_unpackhi_epi32(x6, x7); + x0 = _mm_unpacklo_epi64(x8, x9); + x1 = _mm_unpacklo_epi64(x10, x11); + x2 = _mm_unpackhi_epi64(x8, x9); + x3 = _mm_unpackhi_epi64(x10, x11); + x4 = _mm_unpacklo_epi64(x12, x13); + x5 = _mm_unpacklo_epi64(x14, x15); + x6 = _mm_unpackhi_epi64(x12, x13); + x7 = _mm_unpackhi_epi64(x14, x15); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 240)); + x0 = _mm_xor_si128(x0, x8); + x1 = _mm_xor_si128(x1, x9); + x2 = _mm_xor_si128(x2, x10); + x3 = _mm_xor_si128(x3, x11); + x4 = _mm_xor_si128(x4, x12); + x5 = _mm_xor_si128(x5, x13); + x6 = _mm_xor_si128(x6, x14); + x7 = _mm_xor_si128(x7, x15); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 96), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 112), x3); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 160), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 176), x5); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 224), x6); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 240), x7); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 192)); + rsi = (word64)(rsi + 0x100); + rdx = (word64)(rdx + 0x100); + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR(r13, 0))); + rcx = (word32)((word32)rcx - 0x100); + _mm_storeu_si128((__m128i*)WC_PW(r9, 192), x12); + if (((word32)rcx) < (0x100)) { + goto L_chacha20_avx512vl_done128; + } + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 48)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 80)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 112)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 128)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 144)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 160)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 176)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 192)); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 208)); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 224)); + x15 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 240)); + goto L_chacha20_avx512vl_start128; +L_chacha20_avx512vl_done128: + rax = (word32)((word32)rax << 2); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + (word32)rax); +L_chacha20_avx512vl_end128: + if (((word32)rcx) == (0)) { + goto L_chacha20_avx512vl_last_done; + } + x0 = _mm_set1_epi32((int)WC_L32(rdi, 0)); + x1 = _mm_set1_epi32((int)WC_L32(rdi, 4)); + x2 = _mm_set1_epi32((int)WC_L32(rdi, 8)); + x3 = _mm_set1_epi32((int)WC_L32(rdi, 12)); + x4 = _mm_set1_epi32((int)WC_L32(rdi, 16)); + x5 = _mm_set1_epi32((int)WC_L32(rdi, 20)); + x6 = _mm_set1_epi32((int)WC_L32(rdi, 24)); + x7 = _mm_set1_epi32((int)WC_L32(rdi, 28)); + x8 = _mm_set1_epi32((int)WC_L32(rdi, 32)); + x9 = _mm_set1_epi32((int)WC_L32(rdi, 36)); + x10 = _mm_set1_epi32((int)WC_L32(rdi, 40)); + x11 = _mm_set1_epi32((int)WC_L32(rdi, 44)); + x12 = _mm_set1_epi32((int)WC_L32(rdi, 48)); + x13 = _mm_set1_epi32((int)WC_L32(rdi, 52)); + x14 = _mm_set1_epi32((int)WC_L32(rdi, 56)); + x15 = _mm_set1_epi32((int)WC_L32(rdi, 60)); + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR(r12, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r9, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r9, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(r9, 48), x3); + _mm_storeu_si128((__m128i*)WC_PW(r9, 64), x4); + _mm_storeu_si128((__m128i*)WC_PW(r9, 80), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 96), x6); + _mm_storeu_si128((__m128i*)WC_PW(r9, 112), x7); + _mm_storeu_si128((__m128i*)WC_PW(r9, 128), x8); + _mm_storeu_si128((__m128i*)WC_PW(r9, 144), x9); + _mm_storeu_si128((__m128i*)WC_PW(r9, 160), x10); + _mm_storeu_si128((__m128i*)WC_PW(r9, 176), x11); + _mm_storeu_si128((__m128i*)WC_PW(r9, 192), x12); + _mm_storeu_si128((__m128i*)WC_PW(r9, 208), x13); + _mm_storeu_si128((__m128i*)WC_PW(r9, 224), x14); + _mm_storeu_si128((__m128i*)WC_PW(r9, 240), x15); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_chacha20_avx512vl_last_round: + x0 = _mm_add_epi32(x0, x4); + x1 = _mm_add_epi32(x1, x5); + x2 = _mm_add_epi32(x2, x6); + x3 = _mm_add_epi32(x3, x7); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + x12 = _mm_rol_epi32(x12, 16); + x13 = _mm_rol_epi32(x13, 16); + x14 = _mm_rol_epi32(x14, 16); + x15 = _mm_rol_epi32(x15, 16); + x8 = _mm_add_epi32(x8, x12); + x9 = _mm_add_epi32(x9, x13); + x10 = _mm_add_epi32(x10, x14); + x11 = _mm_add_epi32(x11, x15); + x4 = _mm_xor_si128(x4, x8); + x5 = _mm_xor_si128(x5, x9); + x6 = _mm_xor_si128(x6, x10); + x7 = _mm_xor_si128(x7, x11); + x4 = _mm_rol_epi32(x4, 12); + x5 = _mm_rol_epi32(x5, 12); + x6 = _mm_rol_epi32(x6, 12); + x7 = _mm_rol_epi32(x7, 12); + x0 = _mm_add_epi32(x0, x4); + x1 = _mm_add_epi32(x1, x5); + x2 = _mm_add_epi32(x2, x6); + x3 = _mm_add_epi32(x3, x7); + x12 = _mm_xor_si128(x12, x0); + x13 = _mm_xor_si128(x13, x1); + x14 = _mm_xor_si128(x14, x2); + x15 = _mm_xor_si128(x15, x3); + x12 = _mm_rol_epi32(x12, 8); + x13 = _mm_rol_epi32(x13, 8); + x14 = _mm_rol_epi32(x14, 8); + x15 = _mm_rol_epi32(x15, 8); + x8 = _mm_add_epi32(x8, x12); + x9 = _mm_add_epi32(x9, x13); + x10 = _mm_add_epi32(x10, x14); + x11 = _mm_add_epi32(x11, x15); + x4 = _mm_xor_si128(x4, x8); + x5 = _mm_xor_si128(x5, x9); + x6 = _mm_xor_si128(x6, x10); + x7 = _mm_xor_si128(x7, x11); + x4 = _mm_rol_epi32(x4, 7); + x5 = _mm_rol_epi32(x5, 7); + x6 = _mm_rol_epi32(x6, 7); + x7 = _mm_rol_epi32(x7, 7); + x0 = _mm_add_epi32(x0, x5); + x1 = _mm_add_epi32(x1, x6); + x2 = _mm_add_epi32(x2, x7); + x3 = _mm_add_epi32(x3, x4); + x15 = _mm_xor_si128(x15, x0); + x12 = _mm_xor_si128(x12, x1); + x13 = _mm_xor_si128(x13, x2); + x14 = _mm_xor_si128(x14, x3); + x15 = _mm_rol_epi32(x15, 16); + x12 = _mm_rol_epi32(x12, 16); + x13 = _mm_rol_epi32(x13, 16); + x14 = _mm_rol_epi32(x14, 16); + x10 = _mm_add_epi32(x10, x15); + x11 = _mm_add_epi32(x11, x12); + x8 = _mm_add_epi32(x8, x13); + x9 = _mm_add_epi32(x9, x14); + x5 = _mm_xor_si128(x5, x10); + x6 = _mm_xor_si128(x6, x11); + x7 = _mm_xor_si128(x7, x8); + x4 = _mm_xor_si128(x4, x9); + x5 = _mm_rol_epi32(x5, 12); + x6 = _mm_rol_epi32(x6, 12); + x7 = _mm_rol_epi32(x7, 12); + x4 = _mm_rol_epi32(x4, 12); + x0 = _mm_add_epi32(x0, x5); + x1 = _mm_add_epi32(x1, x6); + x2 = _mm_add_epi32(x2, x7); + x3 = _mm_add_epi32(x3, x4); + x15 = _mm_xor_si128(x15, x0); + x12 = _mm_xor_si128(x12, x1); + x13 = _mm_xor_si128(x13, x2); + x14 = _mm_xor_si128(x14, x3); + x15 = _mm_rol_epi32(x15, 8); + x12 = _mm_rol_epi32(x12, 8); + x13 = _mm_rol_epi32(x13, 8); + x14 = _mm_rol_epi32(x14, 8); + x10 = _mm_add_epi32(x10, x15); + x11 = _mm_add_epi32(x11, x12); + x8 = _mm_add_epi32(x8, x13); + x9 = _mm_add_epi32(x9, x14); + x5 = _mm_xor_si128(x5, x10); + x6 = _mm_xor_si128(x6, x11); + x7 = _mm_xor_si128(x7, x8); + x4 = _mm_xor_si128(x4, x9); + x5 = _mm_rol_epi32(x5, 7); + x6 = _mm_rol_epi32(x6, 7); + x7 = _mm_rol_epi32(x7, 7); + x4 = _mm_rol_epi32(x4, 7); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 - 1) & 0xff); + if (((byte)r8) != (0)) { + goto L_chacha20_avx512vl_last_round; + } + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + x1 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(r9, 16))); + x2 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(r9, 32))); + x3 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(r9, 48))); + x4 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(r9, 64))); + x5 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(r9, 80))); + x6 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(r9, 96))); + x7 = _mm_add_epi32(x7, _mm_loadu_si128((const __m128i*)WC_PR(r9, 112))); + x8 = _mm_add_epi32(x8, _mm_loadu_si128((const __m128i*)WC_PR(r9, 128))); + x9 = _mm_add_epi32(x9, _mm_loadu_si128((const __m128i*)WC_PR(r9, 144))); + x10 = _mm_add_epi32(x10, _mm_loadu_si128((const __m128i*)WC_PR(r9, 160))); + x11 = _mm_add_epi32(x11, _mm_loadu_si128((const __m128i*)WC_PR(r9, 176))); + x12 = _mm_add_epi32(x12, _mm_loadu_si128((const __m128i*)WC_PR(r9, 192))); + x13 = _mm_add_epi32(x13, _mm_loadu_si128((const __m128i*)WC_PR(r9, 208))); + x14 = _mm_add_epi32(x14, _mm_loadu_si128((const __m128i*)WC_PR(r9, 224))); + x15 = _mm_add_epi32(x15, _mm_loadu_si128((const __m128i*)WC_PR(r9, 240))); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(r10, 16), x9); + _mm_storeu_si128((__m128i*)WC_PW(r10, 32), x10); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x11); + _mm_storeu_si128((__m128i*)WC_PW(r10, 64), x12); + _mm_storeu_si128((__m128i*)WC_PW(r10, 80), x13); + _mm_storeu_si128((__m128i*)WC_PW(r10, 96), x14); + _mm_storeu_si128((__m128i*)WC_PW(r10, 112), x15); + x8 = _mm_unpacklo_epi32(x0, x1); + x9 = _mm_unpacklo_epi32(x2, x3); + x12 = _mm_unpackhi_epi32(x0, x1); + x13 = _mm_unpackhi_epi32(x2, x3); + x10 = _mm_unpacklo_epi32(x4, x5); + x11 = _mm_unpacklo_epi32(x6, x7); + x14 = _mm_unpackhi_epi32(x4, x5); + x15 = _mm_unpackhi_epi32(x6, x7); + x0 = _mm_unpacklo_epi64(x8, x9); + x1 = _mm_unpacklo_epi64(x10, x11); + x2 = _mm_unpackhi_epi64(x8, x9); + x3 = _mm_unpackhi_epi64(x10, x11); + x4 = _mm_unpacklo_epi64(x12, x13); + x5 = _mm_unpacklo_epi64(x14, x15); + x6 = _mm_unpackhi_epi64(x12, x13); + x7 = _mm_unpackhi_epi64(x14, x15); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(r9, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(r9, 64), x2); + _mm_storeu_si128((__m128i*)WC_PW(r9, 80), x3); + _mm_storeu_si128((__m128i*)WC_PW(r9, 128), x4); + _mm_storeu_si128((__m128i*)WC_PW(r9, 144), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 192), x6); + _mm_storeu_si128((__m128i*)WC_PW(r9, 208), x7); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 16)); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 48)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 80)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(r10, 112)); + x8 = _mm_unpacklo_epi32(x0, x1); + x9 = _mm_unpacklo_epi32(x2, x3); + x12 = _mm_unpackhi_epi32(x0, x1); + x13 = _mm_unpackhi_epi32(x2, x3); + x10 = _mm_unpacklo_epi32(x4, x5); + x11 = _mm_unpacklo_epi32(x6, x7); + x14 = _mm_unpackhi_epi32(x4, x5); + x15 = _mm_unpackhi_epi32(x6, x7); + x0 = _mm_unpacklo_epi64(x8, x9); + x1 = _mm_unpacklo_epi64(x10, x11); + x2 = _mm_unpackhi_epi64(x8, x9); + x3 = _mm_unpackhi_epi64(x10, x11); + x4 = _mm_unpacklo_epi64(x12, x13); + x5 = _mm_unpacklo_epi64(x14, x15); + x6 = _mm_unpackhi_epi64(x12, x13); + x7 = _mm_unpackhi_epi64(x14, x15); + _mm_storeu_si128((__m128i*)WC_PW(r9, 32), x0); + _mm_storeu_si128((__m128i*)WC_PW(r9, 48), x1); + _mm_storeu_si128((__m128i*)WC_PW(r9, 96), x2); + _mm_storeu_si128((__m128i*)WC_PW(r9, 112), x3); + _mm_storeu_si128((__m128i*)WC_PW(r9, 160), x4); + _mm_storeu_si128((__m128i*)WC_PW(r9, 176), x5); + _mm_storeu_si128((__m128i*)WC_PW(r9, 224), x6); + _mm_storeu_si128((__m128i*)WC_PW(r9, 240), x7); + if (((word32)rcx) < (0x40)) { + goto L_chacha20_avx512vl_last_fdone; + } +L_chacha20_avx512vl_last_fstart: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 16), x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 32))); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 32), x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(r9, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 48), x0); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 1); + rcx = (word32)((word32)rcx - 0x40); + rsi = (word64)(rsi + 0x40); + rdx = (word64)(rdx + 0x40); + r9 = (word64)(r9 + 0x40); + if (((word32)rcx) >= (0x40)) { + goto L_chacha20_avx512vl_last_fstart; + } +L_chacha20_avx512vl_last_fdone: + if (((word32)rcx) == (0)) { + goto L_chacha20_avx512vl_last_done; + } + r10 = (word64)(rdi + 80); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 0)); + _mm_storeu_si128((__m128i*)WC_PW(r10, 0), x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 16)); + _mm_storeu_si128((__m128i*)WC_PW(r10, 16), x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 32)); + _mm_storeu_si128((__m128i*)WC_PW(r10, 32), x0); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(r9, 48)); + _mm_storeu_si128((__m128i*)WC_PW(r10, 48), x0); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 1); + r8 = (word32)((word32)rcx); + r11 = (word64)(0); + r8 = (word32)((word32)r8 & 7); + if (((word32)r8) == (0)) { + goto L_chacha20_avx512vl_last_start64; + } +L_chacha20_avx512vl_last_start8: + rax = (word32)((word32)WC_L8(r10, r11)); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax ^ WC_L8(rsi, + r11)) & 0xff); + WC_S8(rdx, r11) = (byte)((byte)rax); + r11 = (word32)((word32)r11 + 1); + zf1 = (word32)r11; + zf2 = (word32)r8; + if (((word32)r11) != ((word32)r8)) { + goto L_chacha20_avx512vl_last_start8; + } + if ((zf1) == (zf2)) { + goto L_chacha20_avx512vl_last_end64; + } +L_chacha20_avx512vl_last_start64: + rax = (word64)(WC_L64(r10, r11)); + rax = (word64)(rax ^ WC_L64(rsi, r11)); + WC_S64(rdx, r11) = (word64)(rax); + r11 = (word32)((word32)r11 + 8); +L_chacha20_avx512vl_last_end64: + if (((word32)r11) != ((word32)rcx)) { + goto L_chacha20_avx512vl_last_start64; + } + r8 = (word32)(0x40); + r8 = (word32)((word32)r8 - (word32)r11); + WC_S32(rdi, 76) = (word32)((word32)r8); +L_chacha20_avx512vl_last_done: + ; +} + +#endif /* HAVE_INTEL_AVX512 */ +#ifdef HAVE_INTEL_AVX512 +XALIGNED(32) static const word64 L_chacha20_avx512_add[] WC_X64I_UNUSED = { + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000500000004ULL, 0x0000000700000006ULL, + 0x0000000900000008ULL, 0x0000000b0000000aULL, + 0x0000000d0000000cULL, 0x0000000f0000000eULL, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void chacha_encrypt_avx512(ChaCha* ctx, const byte* m, byte* c, + word32 bytes) +{ + word64 rdi, rsi, rdx, rcx, rsp, r8, rax = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), + z8 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), + z12 = _mm512_setzero_si512(), z13 = _mm512_setzero_si512(), + z14 = _mm512_setzero_si512(), z15 = _mm512_setzero_si512(), + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(), + z22 = _mm512_setzero_si512(), z23 = _mm512_setzero_si512(), + z24 = _mm512_setzero_si512(), z25 = _mm512_setzero_si512(), + z26 = _mm512_setzero_si512(), z27 = _mm512_setzero_si512(), + z28 = _mm512_setzero_si512(), z29 = _mm512_setzero_si512(), + z30 = _mm512_setzero_si512(), z31 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rdx = (word64)(size_t)c; + rcx = (word64)(word32)bytes; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 8); + r8 = (word64)((word64)(size_t)L_chacha20_avx512_add); + if (((word32)rcx) < (0x400)) { + goto L_chacha20_avx512_end512; + } +L_chacha20_avx512_start512: + z0 = _mm512_set1_epi32((int)WC_L32(rdi, 0)); + z1 = _mm512_set1_epi32((int)WC_L32(rdi, 4)); + z2 = _mm512_set1_epi32((int)WC_L32(rdi, 8)); + z3 = _mm512_set1_epi32((int)WC_L32(rdi, 12)); + z4 = _mm512_set1_epi32((int)WC_L32(rdi, 16)); + z5 = _mm512_set1_epi32((int)WC_L32(rdi, 20)); + z6 = _mm512_set1_epi32((int)WC_L32(rdi, 24)); + z7 = _mm512_set1_epi32((int)WC_L32(rdi, 28)); + z8 = _mm512_set1_epi32((int)WC_L32(rdi, 32)); + z9 = _mm512_set1_epi32((int)WC_L32(rdi, 36)); + z10 = _mm512_set1_epi32((int)WC_L32(rdi, 40)); + z11 = _mm512_set1_epi32((int)WC_L32(rdi, 44)); + z13 = _mm512_set1_epi32((int)WC_L32(rdi, 52)); + z14 = _mm512_set1_epi32((int)WC_L32(rdi, 56)); + z15 = _mm512_set1_epi32((int)WC_L32(rdi, 60)); + z12 = _mm512_set1_epi32((int)WC_L32(rdi, 48)); + z12 = _mm512_add_epi32(z12, _mm512_loadu_si512((const void*)WC_PR(r8, 0))); + rax = (rax & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_chacha20_avx512_loop512: + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax - 1) & 0xff); + if (((byte)rax) != (0)) { + goto L_chacha20_avx512_loop512; + } + z0 = _mm512_add_epi32(z0, _mm512_set1_epi32((int)WC_L32(rdi, 0))); + z1 = _mm512_add_epi32(z1, _mm512_set1_epi32((int)WC_L32(rdi, 4))); + z2 = _mm512_add_epi32(z2, _mm512_set1_epi32((int)WC_L32(rdi, 8))); + z3 = _mm512_add_epi32(z3, _mm512_set1_epi32((int)WC_L32(rdi, 12))); + z4 = _mm512_add_epi32(z4, _mm512_set1_epi32((int)WC_L32(rdi, 16))); + z5 = _mm512_add_epi32(z5, _mm512_set1_epi32((int)WC_L32(rdi, 20))); + z6 = _mm512_add_epi32(z6, _mm512_set1_epi32((int)WC_L32(rdi, 24))); + z7 = _mm512_add_epi32(z7, _mm512_set1_epi32((int)WC_L32(rdi, 28))); + z8 = _mm512_add_epi32(z8, _mm512_set1_epi32((int)WC_L32(rdi, 32))); + z9 = _mm512_add_epi32(z9, _mm512_set1_epi32((int)WC_L32(rdi, 36))); + z10 = _mm512_add_epi32(z10, _mm512_set1_epi32((int)WC_L32(rdi, 40))); + z11 = _mm512_add_epi32(z11, _mm512_set1_epi32((int)WC_L32(rdi, 44))); + z13 = _mm512_add_epi32(z13, _mm512_set1_epi32((int)WC_L32(rdi, 52))); + z14 = _mm512_add_epi32(z14, _mm512_set1_epi32((int)WC_L32(rdi, 56))); + z15 = _mm512_add_epi32(z15, _mm512_set1_epi32((int)WC_L32(rdi, 60))); + z16 = _mm512_set1_epi32((int)WC_L32(rdi, 48)); + z16 = _mm512_add_epi32(z16, _mm512_loadu_si512((const void*)WC_PR(r8, 0))); + z12 = _mm512_add_epi32(z12, z16); + z16 = _mm512_unpacklo_epi32(z0, z1); + z17 = _mm512_unpackhi_epi32(z0, z1); + z18 = _mm512_unpacklo_epi32(z2, z3); + z19 = _mm512_unpackhi_epi32(z2, z3); + z20 = _mm512_unpacklo_epi32(z4, z5); + z21 = _mm512_unpackhi_epi32(z4, z5); + z22 = _mm512_unpacklo_epi32(z6, z7); + z23 = _mm512_unpackhi_epi32(z6, z7); + z24 = _mm512_unpacklo_epi32(z8, z9); + z25 = _mm512_unpackhi_epi32(z8, z9); + z26 = _mm512_unpacklo_epi32(z10, z11); + z27 = _mm512_unpackhi_epi32(z10, z11); + z28 = _mm512_unpacklo_epi32(z12, z13); + z29 = _mm512_unpackhi_epi32(z12, z13); + z30 = _mm512_unpacklo_epi32(z14, z15); + z31 = _mm512_unpackhi_epi32(z14, z15); + z0 = _mm512_unpacklo_epi64(z16, z18); + z1 = _mm512_unpackhi_epi64(z16, z18); + z2 = _mm512_unpacklo_epi64(z17, z19); + z3 = _mm512_unpackhi_epi64(z17, z19); + z4 = _mm512_unpacklo_epi64(z20, z22); + z5 = _mm512_unpackhi_epi64(z20, z22); + z6 = _mm512_unpacklo_epi64(z21, z23); + z7 = _mm512_unpackhi_epi64(z21, z23); + z8 = _mm512_unpacklo_epi64(z24, z26); + z9 = _mm512_unpackhi_epi64(z24, z26); + z10 = _mm512_unpacklo_epi64(z25, z27); + z11 = _mm512_unpackhi_epi64(z25, z27); + z12 = _mm512_unpacklo_epi64(z28, z30); + z13 = _mm512_unpackhi_epi64(z28, z30); + z14 = _mm512_unpacklo_epi64(z29, z31); + z15 = _mm512_unpackhi_epi64(z29, z31); + z16 = _mm512_shuffle_i32x4(z0, z4, 0x44); + z17 = _mm512_shuffle_i32x4(z0, z4, 0xee); + z18 = _mm512_shuffle_i32x4(z8, z12, 0x44); + z19 = _mm512_shuffle_i32x4(z8, z12, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rsi, 0))); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rsi, + 256))); + _mm512_storeu_si512((void*)WC_PW(rdx, 256), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rsi, + 512))); + _mm512_storeu_si512((void*)WC_PW(rdx, 512), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rsi, + 768))); + _mm512_storeu_si512((void*)WC_PW(rdx, 768), z23); + z16 = _mm512_shuffle_i32x4(z1, z5, 0x44); + z17 = _mm512_shuffle_i32x4(z1, z5, 0xee); + z18 = _mm512_shuffle_i32x4(z9, z13, 0x44); + z19 = _mm512_shuffle_i32x4(z9, z13, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rsi, + 64))); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rsi, + 320))); + _mm512_storeu_si512((void*)WC_PW(rdx, 320), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rsi, + 576))); + _mm512_storeu_si512((void*)WC_PW(rdx, 576), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rsi, + 832))); + _mm512_storeu_si512((void*)WC_PW(rdx, 832), z23); + z16 = _mm512_shuffle_i32x4(z2, z6, 0x44); + z17 = _mm512_shuffle_i32x4(z2, z6, 0xee); + z18 = _mm512_shuffle_i32x4(z10, z14, 0x44); + z19 = _mm512_shuffle_i32x4(z10, z14, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rsi, + 128))); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rsi, + 384))); + _mm512_storeu_si512((void*)WC_PW(rdx, 384), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rsi, + 640))); + _mm512_storeu_si512((void*)WC_PW(rdx, 640), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rsi, + 896))); + _mm512_storeu_si512((void*)WC_PW(rdx, 896), z23); + z16 = _mm512_shuffle_i32x4(z3, z7, 0x44); + z17 = _mm512_shuffle_i32x4(z3, z7, 0xee); + z18 = _mm512_shuffle_i32x4(z11, z15, 0x44); + z19 = _mm512_shuffle_i32x4(z11, z15, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rsi, + 192))); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rsi, + 448))); + _mm512_storeu_si512((void*)WC_PW(rdx, 448), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rsi, + 704))); + _mm512_storeu_si512((void*)WC_PW(rdx, 704), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rsi, + 960))); + _mm512_storeu_si512((void*)WC_PW(rdx, 960), z23); + rsi = (word64)(rsi + 0x400); + rdx = (word64)(rdx + 0x400); + WC_S32(rdi, 48) = (word32)(WC_L32(rdi, 48) + 0x10); + rcx = (word32)((word32)rcx - 0x400); + if (((word32)rcx) >= (0x400)) { + goto L_chacha20_avx512_start512; + } +L_chacha20_avx512_end512: + (void)chacha_encrypt_avx2((ChaCha*)(size_t)rdi, (const byte*)(size_t)rsi, ( + byte*)(size_t)rdx, (word32)rcx); +} + +#endif /* HAVE_INTEL_AVX512 */ +#ifdef HAVE_INTEL_AVX512 +XALIGNED(16) static const word64 L_chacha20_poly1305_add[] WC_X64I_UNUSED = { + 0x0000000100000000ULL, 0x0000000300000002ULL, +}; + +XALIGNED(16) static const word64 L_chacha20_poly1305_four[] WC_X64I_UNUSED = { + 0x0000000400000004ULL, 0x0000000400000004ULL, +}; + +XALIGNED(32) static const word64 L_chacha20_poly1305_mask[] WC_X64I_UNUSED = { + 0x0000000003ffffffULL, 0x0000000003ffffffULL, + 0x0000000003ffffffULL, 0x0000000003ffffffULL, +}; + +XALIGNED(32) static const word64 L_chacha20_poly1305_hibit[] WC_X64I_UNUSED = { + 0x0000000001000000ULL, 0x0000000001000000ULL, + 0x0000000001000000ULL, 0x0000000001000000ULL, +}; + +WC_X64I_TARGET("avx512f,avx512vl") +WOLFSSL_LOCAL void chacha20_poly1305_avx512(ChaCha* chacha, Poly1305* poly, + const byte* m, byte* c, word32 bytes) +{ + word64 rdi, rsi, rdx, rcx, r8, rsp, r10, r11, r12, r13, r14, r15, rbx, + rbp, r9, rax = 0; + __m128i x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, + x29, x30, x31; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + XALIGNED(32) WC_X64I_SLOT stk[92]; + + rdi = (word64)(size_t)chacha; + rsi = (word64)(size_t)poly; + rdx = (word64)(size_t)m; + rcx = (word64)(size_t)c; + r8 = (word64)(word32)bytes; + + rsp = (word64)(size_t)stk + 736; + rsp = (word64)(rsp - 720); + r10 = (word64)((word64)(size_t)L_chacha20_poly1305_add); + r11 = (word64)((word64)(size_t)L_chacha20_poly1305_four); + r12 = (word64)((word64)(size_t)L_chacha20_poly1305_mask); + r13 = (word64)((word64)(size_t)L_chacha20_poly1305_hibit); + r14 = (word64)(rsp); + r14 = (word64)(r14 + 0xf); + r14 = (word64)(r14 & -16); + r15 = (word64)(rsp + 272); + r15 = (word64)(r15 + 0xf); + r15 = (word64)(r15 & -16); + rbx = (word64)(rsp + 416); + rbp = (word64)(rsp + 576); + y15 = _mm256_zextsi128_si256(_mm_setzero_si128()); + r9 = (word64)(rsi + 64); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 128)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y5 = _mm256_permute4x64_epi64(y13, 0); + y14 = _mm256_srli_epi64(y13, 32); + y7 = _mm256_permute4x64_epi64(y13, 0x55); + y9 = _mm256_permute4x64_epi64(y13, 0xaa); + y6 = _mm256_permute4x64_epi64(y14, 0); + y8 = _mm256_permute4x64_epi64(y14, 0x55); + y10 = _mm256_slli_epi32(y6, 2); + y11 = _mm256_slli_epi32(y7, 2); + y12 = _mm256_slli_epi32(y8, 2); + y13 = _mm256_slli_epi32(y9, 2); + y10 = _mm256_add_epi64(y6, y10); + y11 = _mm256_add_epi64(y7, y11); + y12 = _mm256_add_epi64(y8, y12); + y13 = _mm256_add_epi64(y9, y13); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 0), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 32), y11); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 64), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 96), y13); + _mm256_storeu_si256((__m256i*)WC_PW(rbx, 0), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rbx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rbx, 64), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rbx, 96), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rbx, 128), y9); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 0)); + x16 = _mm_set1_epi32((int)WC_L32(rdi, 0)); + x17 = _mm_set1_epi32((int)WC_L32(rdi, 4)); + x18 = _mm_set1_epi32((int)WC_L32(rdi, 8)); + x19 = _mm_set1_epi32((int)WC_L32(rdi, 12)); + x20 = _mm_set1_epi32((int)WC_L32(rdi, 16)); + x21 = _mm_set1_epi32((int)WC_L32(rdi, 20)); + x22 = _mm_set1_epi32((int)WC_L32(rdi, 24)); + x23 = _mm_set1_epi32((int)WC_L32(rdi, 28)); + x24 = _mm_set1_epi32((int)WC_L32(rdi, 32)); + x25 = _mm_set1_epi32((int)WC_L32(rdi, 36)); + x26 = _mm_set1_epi32((int)WC_L32(rdi, 40)); + x27 = _mm_set1_epi32((int)WC_L32(rdi, 44)); + x28 = _mm_set1_epi32((int)WC_L32(rdi, 48)); + x29 = _mm_set1_epi32((int)WC_L32(rdi, 52)); + x30 = _mm_set1_epi32((int)WC_L32(rdi, 56)); + x31 = _mm_set1_epi32((int)WC_L32(rdi, 60)); + x28 = _mm_add_epi32(x28, _mm_loadu_si128((const __m128i*)WC_PR(r10, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r14, 0), x16); + _mm_storeu_si128((__m128i*)WC_PW(r14, 16), x17); + _mm_storeu_si128((__m128i*)WC_PW(r14, 32), x18); + _mm_storeu_si128((__m128i*)WC_PW(r14, 48), x19); + _mm_storeu_si128((__m128i*)WC_PW(r14, 64), x20); + _mm_storeu_si128((__m128i*)WC_PW(r14, 80), x21); + _mm_storeu_si128((__m128i*)WC_PW(r14, 96), x22); + _mm_storeu_si128((__m128i*)WC_PW(r14, 112), x23); + _mm_storeu_si128((__m128i*)WC_PW(r14, 128), x24); + _mm_storeu_si128((__m128i*)WC_PW(r14, 144), x25); + _mm_storeu_si128((__m128i*)WC_PW(r14, 160), x26); + _mm_storeu_si128((__m128i*)WC_PW(r14, 176), x27); + _mm_storeu_si128((__m128i*)WC_PW(r14, 192), x28); + _mm_storeu_si128((__m128i*)WC_PW(r14, 208), x29); + _mm_storeu_si128((__m128i*)WC_PW(r14, 224), x30); + _mm_storeu_si128((__m128i*)WC_PW(r14, 240), x31); + rax = (rax & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_chacha20_poly1305_rounds: + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax - 1) & 0xff); + if (((byte)rax) != (0)) { + goto L_chacha20_poly1305_rounds; + } + x16 = _mm_add_epi32(x16, _mm_loadu_si128((const __m128i*)WC_PR(r14, 0))); + x17 = _mm_add_epi32(x17, _mm_loadu_si128((const __m128i*)WC_PR(r14, 16))); + x18 = _mm_add_epi32(x18, _mm_loadu_si128((const __m128i*)WC_PR(r14, 32))); + x19 = _mm_add_epi32(x19, _mm_loadu_si128((const __m128i*)WC_PR(r14, 48))); + x20 = _mm_add_epi32(x20, _mm_loadu_si128((const __m128i*)WC_PR(r14, 64))); + x21 = _mm_add_epi32(x21, _mm_loadu_si128((const __m128i*)WC_PR(r14, 80))); + x22 = _mm_add_epi32(x22, _mm_loadu_si128((const __m128i*)WC_PR(r14, 96))); + x23 = _mm_add_epi32(x23, _mm_loadu_si128((const __m128i*)WC_PR(r14, 112))); + x24 = _mm_add_epi32(x24, _mm_loadu_si128((const __m128i*)WC_PR(r14, 128))); + x25 = _mm_add_epi32(x25, _mm_loadu_si128((const __m128i*)WC_PR(r14, 144))); + x26 = _mm_add_epi32(x26, _mm_loadu_si128((const __m128i*)WC_PR(r14, 160))); + x27 = _mm_add_epi32(x27, _mm_loadu_si128((const __m128i*)WC_PR(r14, 176))); + x28 = _mm_add_epi32(x28, _mm_loadu_si128((const __m128i*)WC_PR(r14, 192))); + x29 = _mm_add_epi32(x29, _mm_loadu_si128((const __m128i*)WC_PR(r14, 208))); + x30 = _mm_add_epi32(x30, _mm_loadu_si128((const __m128i*)WC_PR(r14, 224))); + x31 = _mm_add_epi32(x31, _mm_loadu_si128((const __m128i*)WC_PR(r14, 240))); + _mm_storeu_si128((__m128i*)WC_PW(r15, 0), x24); + _mm_storeu_si128((__m128i*)WC_PW(r15, 16), x25); + _mm_storeu_si128((__m128i*)WC_PW(r15, 32), x26); + _mm_storeu_si128((__m128i*)WC_PW(r15, 48), x27); + _mm_storeu_si128((__m128i*)WC_PW(r15, 64), x28); + _mm_storeu_si128((__m128i*)WC_PW(r15, 80), x29); + _mm_storeu_si128((__m128i*)WC_PW(r15, 96), x30); + _mm_storeu_si128((__m128i*)WC_PW(r15, 112), x31); + x24 = _mm_unpacklo_epi32(x16, x17); + x25 = _mm_unpacklo_epi32(x18, x19); + x28 = _mm_unpackhi_epi32(x16, x17); + x29 = _mm_unpackhi_epi32(x18, x19); + x26 = _mm_unpacklo_epi32(x20, x21); + x27 = _mm_unpacklo_epi32(x22, x23); + x30 = _mm_unpackhi_epi32(x20, x21); + x31 = _mm_unpackhi_epi32(x22, x23); + x16 = _mm_unpacklo_epi64(x24, x25); + x17 = _mm_unpacklo_epi64(x26, x27); + x18 = _mm_unpackhi_epi64(x24, x25); + x19 = _mm_unpackhi_epi64(x26, x27); + x20 = _mm_unpacklo_epi64(x28, x29); + x21 = _mm_unpacklo_epi64(x30, x31); + x22 = _mm_unpackhi_epi64(x28, x29); + x23 = _mm_unpackhi_epi64(x30, x31); + x24 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x25 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16)); + x26 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64)); + x27 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80)); + x28 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128)); + x29 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144)); + x30 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 192)); + x31 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 208)); + x16 = _mm_xor_si128(x16, x24); + x17 = _mm_xor_si128(x17, x25); + x18 = _mm_xor_si128(x18, x26); + x19 = _mm_xor_si128(x19, x27); + x20 = _mm_xor_si128(x20, x28); + x21 = _mm_xor_si128(x21, x29); + x22 = _mm_xor_si128(x22, x30); + x23 = _mm_xor_si128(x23, x31); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x16); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 16), x17); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 64), x18); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 80), x19); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 128), x20); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 144), x21); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 192), x22); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 208), x23); + x16 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x17 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x18 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x19 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x20 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x21 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x22 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x23 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x24 = _mm_unpacklo_epi32(x16, x17); + x25 = _mm_unpacklo_epi32(x18, x19); + x28 = _mm_unpackhi_epi32(x16, x17); + x29 = _mm_unpackhi_epi32(x18, x19); + x26 = _mm_unpacklo_epi32(x20, x21); + x27 = _mm_unpacklo_epi32(x22, x23); + x30 = _mm_unpackhi_epi32(x20, x21); + x31 = _mm_unpackhi_epi32(x22, x23); + x16 = _mm_unpacklo_epi64(x24, x25); + x17 = _mm_unpacklo_epi64(x26, x27); + x18 = _mm_unpackhi_epi64(x24, x25); + x19 = _mm_unpackhi_epi64(x26, x27); + x20 = _mm_unpacklo_epi64(x28, x29); + x21 = _mm_unpacklo_epi64(x30, x31); + x22 = _mm_unpackhi_epi64(x28, x29); + x23 = _mm_unpackhi_epi64(x30, x31); + x24 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32)); + x25 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48)); + x26 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96)); + x27 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112)); + x28 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 160)); + x29 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 176)); + x30 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 224)); + x31 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 240)); + x16 = _mm_xor_si128(x16, x24); + x17 = _mm_xor_si128(x17, x25); + x18 = _mm_xor_si128(x18, x26); + x19 = _mm_xor_si128(x19, x27); + x20 = _mm_xor_si128(x20, x28); + x21 = _mm_xor_si128(x21, x29); + x22 = _mm_xor_si128(x22, x30); + x23 = _mm_xor_si128(x23, x31); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 32), x16); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 48), x17); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 96), x18); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 112), x19); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 160), x20); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 176), x21); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 224), x22); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 240), x23); + x28 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 192)); + x28 = _mm_add_epi32(x28, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r14, 192), x28); + x16 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 0)); + x17 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 16)); + x18 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 32)); + x19 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 48)); + x20 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 64)); + x21 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 80)); + x22 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 96)); + x23 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 112)); + x24 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 128)); + x25 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 144)); + x26 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 160)); + x27 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 176)); + x28 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 192)); + x29 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 208)); + x30 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 224)); + x31 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 240)); + rdx = (word64)(rdx + 0x100); + rcx = (word64)(rcx + 0x100); + r8 = (word32)((word32)r8 - 0x100); + if (((word32)r8) == (0)) { + goto L_chacha20_poly1305_epilogue; + } +L_chacha20_poly1305_start: + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -256)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -224)); + y7 = _mm256_permute2x128_si256(y5, y6, 0x20); + y5 = _mm256_permute2x128_si256(y5, y6, 0x31); + y6 = _mm256_unpacklo_epi32(y7, y5); + y8 = _mm256_unpackhi_epi32(y7, y5); + y5 = _mm256_unpacklo_epi32(y6, y15); + y6 = _mm256_unpackhi_epi32(y6, y15); + y7 = _mm256_unpacklo_epi32(y8, y15); + y8 = _mm256_unpackhi_epi32(y8, y15); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + y6 = _mm256_slli_epi64(y6, 6); + y7 = _mm256_slli_epi64(y7, 12); + y8 = _mm256_slli_epi64(y8, 18); + y10 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi64(y10, y5); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y11 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y6 = _mm256_add_epi64(y11, y6); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y12 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y7 = _mm256_add_epi64(y12, y7); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y8 = _mm256_add_epi64(y13, y8); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y9 = _mm256_add_epi64(y13, y9); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -192)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -160)); + y7 = _mm256_permute2x128_si256(y5, y6, 0x20); + y5 = _mm256_permute2x128_si256(y5, y6, 0x31); + y6 = _mm256_unpacklo_epi32(y7, y5); + y8 = _mm256_unpackhi_epi32(y7, y5); + y5 = _mm256_unpacklo_epi32(y6, y15); + y6 = _mm256_unpackhi_epi32(y6, y15); + y7 = _mm256_unpacklo_epi32(y8, y15); + y8 = _mm256_unpackhi_epi32(y8, y15); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + y6 = _mm256_slli_epi64(y6, 6); + y7 = _mm256_slli_epi64(y7, 12); + y8 = _mm256_slli_epi64(y8, 18); + y10 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi64(y10, y5); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y11 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y6 = _mm256_add_epi64(y11, y6); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y12 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y7 = _mm256_add_epi64(y12, y7); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y8 = _mm256_add_epi64(y13, y8); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y9 = _mm256_add_epi64(y13, y9); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -128)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -96)); + y7 = _mm256_permute2x128_si256(y5, y6, 0x20); + y5 = _mm256_permute2x128_si256(y5, y6, 0x31); + y6 = _mm256_unpacklo_epi32(y7, y5); + y8 = _mm256_unpackhi_epi32(y7, y5); + y5 = _mm256_unpacklo_epi32(y6, y15); + y6 = _mm256_unpackhi_epi32(y6, y15); + y7 = _mm256_unpacklo_epi32(y8, y15); + y8 = _mm256_unpackhi_epi32(y8, y15); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + y6 = _mm256_slli_epi64(y6, 6); + y7 = _mm256_slli_epi64(y7, 12); + y8 = _mm256_slli_epi64(y8, 18); + y10 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi64(y10, y5); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y11 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y6 = _mm256_add_epi64(y11, y6); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y12 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y7 = _mm256_add_epi64(y12, y7); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y8 = _mm256_add_epi64(y13, y8); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y9 = _mm256_add_epi64(y13, y9); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -64)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -32)); + y7 = _mm256_permute2x128_si256(y5, y6, 0x20); + y5 = _mm256_permute2x128_si256(y5, y6, 0x31); + y6 = _mm256_unpacklo_epi32(y7, y5); + y8 = _mm256_unpackhi_epi32(y7, y5); + y5 = _mm256_unpacklo_epi32(y6, y15); + y6 = _mm256_unpackhi_epi32(y6, y15); + y7 = _mm256_unpacklo_epi32(y8, y15); + y8 = _mm256_unpackhi_epi32(y8, y15); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + y6 = _mm256_slli_epi64(y6, 6); + y7 = _mm256_slli_epi64(y7, 12); + y8 = _mm256_slli_epi64(y8, 18); + y10 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi64(y10, y5); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y11 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y6 = _mm256_add_epi64(y11, y6); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y12 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y7 = _mm256_add_epi64(y12, y7); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y8 = _mm256_add_epi64(y13, y8); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y9 = _mm256_add_epi64(y13, y9); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x31 = _mm_rol_epi32(x31, 16); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 12); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x16 = _mm_add_epi32(x16, x20); + x17 = _mm_add_epi32(x17, x21); + x18 = _mm_add_epi32(x18, x22); + x19 = _mm_add_epi32(x19, x23); + x28 = _mm_xor_si128(x28, x16); + x29 = _mm_xor_si128(x29, x17); + x30 = _mm_xor_si128(x30, x18); + x31 = _mm_xor_si128(x31, x19); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x31 = _mm_rol_epi32(x31, 8); + x24 = _mm_add_epi32(x24, x28); + x25 = _mm_add_epi32(x25, x29); + x26 = _mm_add_epi32(x26, x30); + x27 = _mm_add_epi32(x27, x31); + x20 = _mm_xor_si128(x20, x24); + x21 = _mm_xor_si128(x21, x25); + x22 = _mm_xor_si128(x22, x26); + x23 = _mm_xor_si128(x23, x27); + x20 = _mm_rol_epi32(x20, 7); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 16); + x28 = _mm_rol_epi32(x28, 16); + x29 = _mm_rol_epi32(x29, 16); + x30 = _mm_rol_epi32(x30, 16); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 12); + x22 = _mm_rol_epi32(x22, 12); + x23 = _mm_rol_epi32(x23, 12); + x20 = _mm_rol_epi32(x20, 12); + x16 = _mm_add_epi32(x16, x21); + x17 = _mm_add_epi32(x17, x22); + x18 = _mm_add_epi32(x18, x23); + x19 = _mm_add_epi32(x19, x20); + x31 = _mm_xor_si128(x31, x16); + x28 = _mm_xor_si128(x28, x17); + x29 = _mm_xor_si128(x29, x18); + x30 = _mm_xor_si128(x30, x19); + x31 = _mm_rol_epi32(x31, 8); + x28 = _mm_rol_epi32(x28, 8); + x29 = _mm_rol_epi32(x29, 8); + x30 = _mm_rol_epi32(x30, 8); + x26 = _mm_add_epi32(x26, x31); + x27 = _mm_add_epi32(x27, x28); + x24 = _mm_add_epi32(x24, x29); + x25 = _mm_add_epi32(x25, x30); + x21 = _mm_xor_si128(x21, x26); + x22 = _mm_xor_si128(x22, x27); + x23 = _mm_xor_si128(x23, x24); + x20 = _mm_xor_si128(x20, x25); + x21 = _mm_rol_epi32(x21, 7); + x22 = _mm_rol_epi32(x22, 7); + x23 = _mm_rol_epi32(x23, 7); + x20 = _mm_rol_epi32(x20, 7); + x16 = _mm_add_epi32(x16, _mm_loadu_si128((const __m128i*)WC_PR(r14, 0))); + x17 = _mm_add_epi32(x17, _mm_loadu_si128((const __m128i*)WC_PR(r14, 16))); + x18 = _mm_add_epi32(x18, _mm_loadu_si128((const __m128i*)WC_PR(r14, 32))); + x19 = _mm_add_epi32(x19, _mm_loadu_si128((const __m128i*)WC_PR(r14, 48))); + x20 = _mm_add_epi32(x20, _mm_loadu_si128((const __m128i*)WC_PR(r14, 64))); + x21 = _mm_add_epi32(x21, _mm_loadu_si128((const __m128i*)WC_PR(r14, 80))); + x22 = _mm_add_epi32(x22, _mm_loadu_si128((const __m128i*)WC_PR(r14, 96))); + x23 = _mm_add_epi32(x23, _mm_loadu_si128((const __m128i*)WC_PR(r14, 112))); + x24 = _mm_add_epi32(x24, _mm_loadu_si128((const __m128i*)WC_PR(r14, 128))); + x25 = _mm_add_epi32(x25, _mm_loadu_si128((const __m128i*)WC_PR(r14, 144))); + x26 = _mm_add_epi32(x26, _mm_loadu_si128((const __m128i*)WC_PR(r14, 160))); + x27 = _mm_add_epi32(x27, _mm_loadu_si128((const __m128i*)WC_PR(r14, 176))); + x28 = _mm_add_epi32(x28, _mm_loadu_si128((const __m128i*)WC_PR(r14, 192))); + x29 = _mm_add_epi32(x29, _mm_loadu_si128((const __m128i*)WC_PR(r14, 208))); + x30 = _mm_add_epi32(x30, _mm_loadu_si128((const __m128i*)WC_PR(r14, 224))); + x31 = _mm_add_epi32(x31, _mm_loadu_si128((const __m128i*)WC_PR(r14, 240))); + _mm_storeu_si128((__m128i*)WC_PW(r15, 0), x24); + _mm_storeu_si128((__m128i*)WC_PW(r15, 16), x25); + _mm_storeu_si128((__m128i*)WC_PW(r15, 32), x26); + _mm_storeu_si128((__m128i*)WC_PW(r15, 48), x27); + _mm_storeu_si128((__m128i*)WC_PW(r15, 64), x28); + _mm_storeu_si128((__m128i*)WC_PW(r15, 80), x29); + _mm_storeu_si128((__m128i*)WC_PW(r15, 96), x30); + _mm_storeu_si128((__m128i*)WC_PW(r15, 112), x31); + x24 = _mm_unpacklo_epi32(x16, x17); + x25 = _mm_unpacklo_epi32(x18, x19); + x28 = _mm_unpackhi_epi32(x16, x17); + x29 = _mm_unpackhi_epi32(x18, x19); + x26 = _mm_unpacklo_epi32(x20, x21); + x27 = _mm_unpacklo_epi32(x22, x23); + x30 = _mm_unpackhi_epi32(x20, x21); + x31 = _mm_unpackhi_epi32(x22, x23); + x16 = _mm_unpacklo_epi64(x24, x25); + x17 = _mm_unpacklo_epi64(x26, x27); + x18 = _mm_unpackhi_epi64(x24, x25); + x19 = _mm_unpackhi_epi64(x26, x27); + x20 = _mm_unpacklo_epi64(x28, x29); + x21 = _mm_unpacklo_epi64(x30, x31); + x22 = _mm_unpackhi_epi64(x28, x29); + x23 = _mm_unpackhi_epi64(x30, x31); + x24 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0)); + x25 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16)); + x26 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64)); + x27 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80)); + x28 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128)); + x29 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144)); + x30 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 192)); + x31 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 208)); + x16 = _mm_xor_si128(x16, x24); + x17 = _mm_xor_si128(x17, x25); + x18 = _mm_xor_si128(x18, x26); + x19 = _mm_xor_si128(x19, x27); + x20 = _mm_xor_si128(x20, x28); + x21 = _mm_xor_si128(x21, x29); + x22 = _mm_xor_si128(x22, x30); + x23 = _mm_xor_si128(x23, x31); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), x16); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 16), x17); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 64), x18); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 80), x19); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 128), x20); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 144), x21); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 192), x22); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 208), x23); + x16 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 0)); + x17 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 16)); + x18 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 32)); + x19 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 48)); + x20 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 64)); + x21 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 80)); + x22 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 96)); + x23 = _mm_loadu_si128((const __m128i*)WC_PR(r15, 112)); + x24 = _mm_unpacklo_epi32(x16, x17); + x25 = _mm_unpacklo_epi32(x18, x19); + x28 = _mm_unpackhi_epi32(x16, x17); + x29 = _mm_unpackhi_epi32(x18, x19); + x26 = _mm_unpacklo_epi32(x20, x21); + x27 = _mm_unpacklo_epi32(x22, x23); + x30 = _mm_unpackhi_epi32(x20, x21); + x31 = _mm_unpackhi_epi32(x22, x23); + x16 = _mm_unpacklo_epi64(x24, x25); + x17 = _mm_unpacklo_epi64(x26, x27); + x18 = _mm_unpackhi_epi64(x24, x25); + x19 = _mm_unpackhi_epi64(x26, x27); + x20 = _mm_unpacklo_epi64(x28, x29); + x21 = _mm_unpacklo_epi64(x30, x31); + x22 = _mm_unpackhi_epi64(x28, x29); + x23 = _mm_unpackhi_epi64(x30, x31); + x24 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32)); + x25 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48)); + x26 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96)); + x27 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112)); + x28 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 160)); + x29 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 176)); + x30 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 224)); + x31 = _mm_loadu_si128((const __m128i*)WC_PR(rdx, 240)); + x16 = _mm_xor_si128(x16, x24); + x17 = _mm_xor_si128(x17, x25); + x18 = _mm_xor_si128(x18, x26); + x19 = _mm_xor_si128(x19, x27); + x20 = _mm_xor_si128(x20, x28); + x21 = _mm_xor_si128(x21, x29); + x22 = _mm_xor_si128(x22, x30); + x23 = _mm_xor_si128(x23, x31); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 32), x16); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 48), x17); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 96), x18); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 112), x19); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 160), x20); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 176), x21); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 224), x22); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 240), x23); + x28 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 192)); + x28 = _mm_add_epi32(x28, _mm_loadu_si128((const __m128i*)WC_PR(r11, 0))); + _mm_storeu_si128((__m128i*)WC_PW(r14, 192), x28); + x16 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 0)); + x17 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 16)); + x18 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 32)); + x19 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 48)); + x20 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 64)); + x21 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 80)); + x22 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 96)); + x23 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 112)); + x24 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 128)); + x25 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 144)); + x26 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 160)); + x27 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 176)); + x28 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 192)); + x29 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 208)); + x30 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 224)); + x31 = _mm_loadu_si128((const __m128i*)WC_PR(r14, 240)); + rdx = (word64)(rdx + 0x100); + rcx = (word64)(rcx + 0x100); + r8 = (word32)((word32)r8 - 0x100); + if (((word32)r8) != (0)) { + goto L_chacha20_poly1305_start; + } +L_chacha20_poly1305_epilogue: + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -256)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -224)); + y7 = _mm256_permute2x128_si256(y5, y6, 0x20); + y5 = _mm256_permute2x128_si256(y5, y6, 0x31); + y6 = _mm256_unpacklo_epi32(y7, y5); + y8 = _mm256_unpackhi_epi32(y7, y5); + y5 = _mm256_unpacklo_epi32(y6, y15); + y6 = _mm256_unpackhi_epi32(y6, y15); + y7 = _mm256_unpacklo_epi32(y8, y15); + y8 = _mm256_unpackhi_epi32(y8, y15); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + y6 = _mm256_slli_epi64(y6, 6); + y7 = _mm256_slli_epi64(y7, 12); + y8 = _mm256_slli_epi64(y8, 18); + y10 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi64(y10, y5); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y11 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y6 = _mm256_add_epi64(y11, y6); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y12 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y7 = _mm256_add_epi64(y12, y7); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y8 = _mm256_add_epi64(y13, y8); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y9 = _mm256_add_epi64(y13, y9); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -192)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -160)); + y7 = _mm256_permute2x128_si256(y5, y6, 0x20); + y5 = _mm256_permute2x128_si256(y5, y6, 0x31); + y6 = _mm256_unpacklo_epi32(y7, y5); + y8 = _mm256_unpackhi_epi32(y7, y5); + y5 = _mm256_unpacklo_epi32(y6, y15); + y6 = _mm256_unpackhi_epi32(y6, y15); + y7 = _mm256_unpacklo_epi32(y8, y15); + y8 = _mm256_unpackhi_epi32(y8, y15); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + y6 = _mm256_slli_epi64(y6, 6); + y7 = _mm256_slli_epi64(y7, 12); + y8 = _mm256_slli_epi64(y8, 18); + y10 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi64(y10, y5); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y11 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y6 = _mm256_add_epi64(y11, y6); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y12 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y7 = _mm256_add_epi64(y12, y7); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y8 = _mm256_add_epi64(y13, y8); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y9 = _mm256_add_epi64(y13, y9); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -128)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -96)); + y7 = _mm256_permute2x128_si256(y5, y6, 0x20); + y5 = _mm256_permute2x128_si256(y5, y6, 0x31); + y6 = _mm256_unpacklo_epi32(y7, y5); + y8 = _mm256_unpackhi_epi32(y7, y5); + y5 = _mm256_unpacklo_epi32(y6, y15); + y6 = _mm256_unpackhi_epi32(y6, y15); + y7 = _mm256_unpacklo_epi32(y8, y15); + y8 = _mm256_unpackhi_epi32(y8, y15); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + y6 = _mm256_slli_epi64(y6, 6); + y7 = _mm256_slli_epi64(y7, 12); + y8 = _mm256_slli_epi64(y8, 18); + y10 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi64(y10, y5); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y11 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y6 = _mm256_add_epi64(y11, y6); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y12 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y7 = _mm256_add_epi64(y12, y7); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y8 = _mm256_add_epi64(y13, y8); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y9 = _mm256_add_epi64(y13, y9); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -64)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -32)); + y7 = _mm256_permute2x128_si256(y5, y6, 0x20); + y5 = _mm256_permute2x128_si256(y5, y6, 0x31); + y6 = _mm256_unpacklo_epi32(y7, y5); + y8 = _mm256_unpackhi_epi32(y7, y5); + y5 = _mm256_unpacklo_epi32(y6, y15); + y6 = _mm256_unpackhi_epi32(y6, y15); + y7 = _mm256_unpacklo_epi32(y8, y15); + y8 = _mm256_unpackhi_epi32(y8, y15); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + y6 = _mm256_slli_epi64(y6, 6); + y7 = _mm256_slli_epi64(y7, 12); + y8 = _mm256_slli_epi64(y8, 18); + y10 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi64(y10, y5); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y11 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + y6 = _mm256_add_epi64(y11, y6); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y12 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y7 = _mm256_add_epi64(y12, y7); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y8 = _mm256_add_epi64(y13, y8); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y9 = _mm256_add_epi64(y13, y9); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 128), y4); + WC_S32(rdi, 48) = (word32)_mm_cvtsi128_si32(x28); +} + +#endif /* HAVE_INTEL_AVX512 */ +#ifdef HAVE_INTEL_AVX512 +XALIGNED(32) static const word64 L_cp512_add[] WC_X64I_UNUSED = { + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000500000004ULL, 0x0000000700000006ULL, + 0x0000000900000008ULL, 0x0000000b0000000aULL, + 0x0000000d0000000cULL, 0x0000000f0000000eULL, +}; + +XALIGNED(32) static const word64 L_cp512_pc[] WC_X64I_UNUSED = { + 0x00000fffffffffffULL, 0x00000fffffffffffULL, + 0x00000fffffffffffULL, 0x00000fffffffffffULL, + 0x00000fffffffffffULL, 0x00000fffffffffffULL, + 0x00000fffffffffffULL, 0x00000fffffffffffULL, + 0x0000000000ffffffULL, 0x0000000000ffffffULL, + 0x0000000000ffffffULL, 0x0000000000ffffffULL, + 0x0000000000ffffffULL, 0x0000000000ffffffULL, + 0x0000000000ffffffULL, 0x0000000000ffffffULL, + 0x0000010000000000ULL, 0x0000010000000000ULL, + 0x0000010000000000ULL, 0x0000010000000000ULL, + 0x0000010000000000ULL, 0x0000010000000000ULL, + 0x0000010000000000ULL, 0x0000010000000000ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000004ULL, 0x0000000000000006ULL, + 0x0000000000000008ULL, 0x000000000000000aULL, + 0x000000000000000cULL, 0x000000000000000eULL, + 0x0000000000000001ULL, 0x0000000000000003ULL, + 0x0000000000000005ULL, 0x0000000000000007ULL, + 0x0000000000000009ULL, 0x000000000000000bULL, + 0x000000000000000dULL, 0x000000000000000fULL, + 0x0000001000000010ULL, 0x0000001000000010ULL, + 0x0000001000000010ULL, 0x0000001000000010ULL, + 0x0000001000000010ULL, 0x0000001000000010ULL, + 0x0000001000000010ULL, 0x0000001000000010ULL, +}; + +XALIGNED(32) static const word64 L_cp512_rx[] WC_X64I_UNUSED = { + 0x000000000000000cULL, 0x0000000000000008ULL, + 0x0000000000000004ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x000000000000000dULL, 0x0000000000000009ULL, + 0x0000000000000005ULL, 0x0000000000000001ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x000000000000000eULL, 0x000000000000000aULL, + 0x0000000000000006ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512ifma") +WOLFSSL_LOCAL void chacha20_poly1305_ifma(ChaCha* chacha, Poly1305* poly, + const byte* m, byte* c, word32 bytes) +{ + word64 rdi, rsi, rdx, rcx, r8, rsp, rax, r10, r9, r11, r12 = 0, r13 = 0, + r14 = 0; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23, z24, z25, z26, z27, + z28, z29, z30, z31; + XALIGNED(32) WC_X64I_SLOT stk[152]; + unsigned char cf; + + rdi = (word64)(size_t)chacha; + rsi = (word64)(size_t)poly; + rdx = (word64)(size_t)m; + rcx = (word64)(size_t)c; + r8 = (word64)(word32)bytes; + + rsp = (word64)(size_t)stk + 1216; + rsp = (word64)(rsp - 1200); + rax = (word64)((word64)(size_t)L_cp512_pc); + r10 = (word64)((word64)(size_t)L_cp512_rx); + r9 = (word64)((word64)(size_t)L_cp512_add); + r11 = (word64)(rsp + 32); + r11 = (word64)(r11 + 0x3f); + r11 = (word64)(r11 & -64); + z16 = _mm512_setzero_si512(); + z17 = _mm512_setzero_si512(); + z18 = _mm512_setzero_si512(); + _mm512_storeu_si512((void*)WC_PW(rsi, 752), z16); + _mm512_storeu_si512((void*)WC_PW(rsi, 816), z17); + _mm512_storeu_si512((void*)WC_PW(rsi, 880), z18); + z19 = _mm512_set1_epi64((long long)WC_L64(rsi, 720)); + z20 = _mm512_set1_epi64((long long)WC_L64(rsi, 728)); + z21 = _mm512_set1_epi64((long long)WC_L64(rsi, 736)); + z30 = _mm512_slli_epi64(z20, 2); + z22 = _mm512_slli_epi64(z20, 4); + z22 = _mm512_add_epi64(z22, z30); + z30 = _mm512_slli_epi64(z21, 2); + z23 = _mm512_slli_epi64(z21, 4); + z23 = _mm512_add_epi64(z23, z30); + z0 = _mm512_set1_epi32((int)WC_L32(rdi, 0)); + z1 = _mm512_set1_epi32((int)WC_L32(rdi, 4)); + z2 = _mm512_set1_epi32((int)WC_L32(rdi, 8)); + z3 = _mm512_set1_epi32((int)WC_L32(rdi, 12)); + z4 = _mm512_set1_epi32((int)WC_L32(rdi, 16)); + z5 = _mm512_set1_epi32((int)WC_L32(rdi, 20)); + z6 = _mm512_set1_epi32((int)WC_L32(rdi, 24)); + z7 = _mm512_set1_epi32((int)WC_L32(rdi, 28)); + z8 = _mm512_set1_epi32((int)WC_L32(rdi, 32)); + z9 = _mm512_set1_epi32((int)WC_L32(rdi, 36)); + z10 = _mm512_set1_epi32((int)WC_L32(rdi, 40)); + z11 = _mm512_set1_epi32((int)WC_L32(rdi, 44)); + z12 = _mm512_set1_epi32((int)WC_L32(rdi, 48)); + z13 = _mm512_set1_epi32((int)WC_L32(rdi, 52)); + z14 = _mm512_set1_epi32((int)WC_L32(rdi, 56)); + z15 = _mm512_set1_epi32((int)WC_L32(rdi, 60)); + z12 = _mm512_add_epi32(z12, _mm512_loadu_si512((const void*)WC_PR(r9, 0))); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z1); + _mm512_storeu_si512((void*)WC_PW(r11, 128), z2); + _mm512_storeu_si512((void*)WC_PW(r11, 192), z3); + _mm512_storeu_si512((void*)WC_PW(r11, 256), z4); + _mm512_storeu_si512((void*)WC_PW(r11, 320), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 384), z6); + _mm512_storeu_si512((void*)WC_PW(r11, 448), z7); + _mm512_storeu_si512((void*)WC_PW(r11, 512), z8); + _mm512_storeu_si512((void*)WC_PW(r11, 576), z9); + _mm512_storeu_si512((void*)WC_PW(r11, 640), z10); + _mm512_storeu_si512((void*)WC_PW(r11, 704), z11); + _mm512_storeu_si512((void*)WC_PW(r11, 768), z12); + _mm512_storeu_si512((void*)WC_PW(r11, 832), z13); + _mm512_storeu_si512((void*)WC_PW(r11, 896), z14); + _mm512_storeu_si512((void*)WC_PW(r11, 960), z15); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(r11, 0))); + z1 = _mm512_add_epi32(z1, _mm512_loadu_si512((const void*)WC_PR(r11, 64))); + z2 = _mm512_add_epi32(z2, _mm512_loadu_si512((const void*)WC_PR(r11, 128))); + z3 = _mm512_add_epi32(z3, _mm512_loadu_si512((const void*)WC_PR(r11, 192))); + z4 = _mm512_add_epi32(z4, _mm512_loadu_si512((const void*)WC_PR(r11, 256))); + z5 = _mm512_add_epi32(z5, _mm512_loadu_si512((const void*)WC_PR(r11, 320))); + z6 = _mm512_add_epi32(z6, _mm512_loadu_si512((const void*)WC_PR(r11, 384))); + z7 = _mm512_add_epi32(z7, _mm512_loadu_si512((const void*)WC_PR(r11, 448))); + z8 = _mm512_add_epi32(z8, _mm512_loadu_si512((const void*)WC_PR(r11, 512))); + z9 = _mm512_add_epi32(z9, _mm512_loadu_si512((const void*)WC_PR(r11, 576))); + z10 = _mm512_add_epi32(z10, _mm512_loadu_si512((const void*)WC_PR(r11, + 640))); + z11 = _mm512_add_epi32(z11, _mm512_loadu_si512((const void*)WC_PR(r11, + 704))); + z12 = _mm512_add_epi32(z12, _mm512_loadu_si512((const void*)WC_PR(r11, + 768))); + z13 = _mm512_add_epi32(z13, _mm512_loadu_si512((const void*)WC_PR(r11, + 832))); + z14 = _mm512_add_epi32(z14, _mm512_loadu_si512((const void*)WC_PR(r11, + 896))); + z15 = _mm512_add_epi32(z15, _mm512_loadu_si512((const void*)WC_PR(r11, + 960))); + z16 = _mm512_unpacklo_epi32(z0, z1); + z17 = _mm512_unpackhi_epi32(z0, z1); + z18 = _mm512_unpacklo_epi32(z2, z3); + z19 = _mm512_unpackhi_epi32(z2, z3); + z20 = _mm512_unpacklo_epi32(z4, z5); + z21 = _mm512_unpackhi_epi32(z4, z5); + z22 = _mm512_unpacklo_epi32(z6, z7); + z23 = _mm512_unpackhi_epi32(z6, z7); + z24 = _mm512_unpacklo_epi32(z8, z9); + z25 = _mm512_unpackhi_epi32(z8, z9); + z26 = _mm512_unpacklo_epi32(z10, z11); + z27 = _mm512_unpackhi_epi32(z10, z11); + z28 = _mm512_unpacklo_epi32(z12, z13); + z29 = _mm512_unpackhi_epi32(z12, z13); + z30 = _mm512_unpacklo_epi32(z14, z15); + z31 = _mm512_unpackhi_epi32(z14, z15); + z0 = _mm512_unpacklo_epi64(z16, z18); + z1 = _mm512_unpackhi_epi64(z16, z18); + z2 = _mm512_unpacklo_epi64(z17, z19); + z3 = _mm512_unpackhi_epi64(z17, z19); + z4 = _mm512_unpacklo_epi64(z20, z22); + z5 = _mm512_unpackhi_epi64(z20, z22); + z6 = _mm512_unpacklo_epi64(z21, z23); + z7 = _mm512_unpackhi_epi64(z21, z23); + z8 = _mm512_unpacklo_epi64(z24, z26); + z9 = _mm512_unpackhi_epi64(z24, z26); + z10 = _mm512_unpacklo_epi64(z25, z27); + z11 = _mm512_unpackhi_epi64(z25, z27); + z12 = _mm512_unpacklo_epi64(z28, z30); + z13 = _mm512_unpackhi_epi64(z28, z30); + z14 = _mm512_unpacklo_epi64(z29, z31); + z15 = _mm512_unpackhi_epi64(z29, z31); + z16 = _mm512_shuffle_i32x4(z0, z4, 0x44); + z17 = _mm512_shuffle_i32x4(z0, z4, 0xee); + z18 = _mm512_shuffle_i32x4(z8, z12, 0x44); + z19 = _mm512_shuffle_i32x4(z8, z12, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, 0))); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 256))); + _mm512_storeu_si512((void*)WC_PW(rcx, 256), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 512))); + _mm512_storeu_si512((void*)WC_PW(rcx, 512), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 768))); + _mm512_storeu_si512((void*)WC_PW(rcx, 768), z23); + z16 = _mm512_shuffle_i32x4(z1, z5, 0x44); + z17 = _mm512_shuffle_i32x4(z1, z5, 0xee); + z18 = _mm512_shuffle_i32x4(z9, z13, 0x44); + z19 = _mm512_shuffle_i32x4(z9, z13, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, + 64))); + _mm512_storeu_si512((void*)WC_PW(rcx, 64), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 320))); + _mm512_storeu_si512((void*)WC_PW(rcx, 320), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 576))); + _mm512_storeu_si512((void*)WC_PW(rcx, 576), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 832))); + _mm512_storeu_si512((void*)WC_PW(rcx, 832), z23); + z16 = _mm512_shuffle_i32x4(z2, z6, 0x44); + z17 = _mm512_shuffle_i32x4(z2, z6, 0xee); + z18 = _mm512_shuffle_i32x4(z10, z14, 0x44); + z19 = _mm512_shuffle_i32x4(z10, z14, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, + 128))); + _mm512_storeu_si512((void*)WC_PW(rcx, 128), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 384))); + _mm512_storeu_si512((void*)WC_PW(rcx, 384), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 640))); + _mm512_storeu_si512((void*)WC_PW(rcx, 640), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 896))); + _mm512_storeu_si512((void*)WC_PW(rcx, 896), z23); + z16 = _mm512_shuffle_i32x4(z3, z7, 0x44); + z17 = _mm512_shuffle_i32x4(z3, z7, 0xee); + z18 = _mm512_shuffle_i32x4(z11, z15, 0x44); + z19 = _mm512_shuffle_i32x4(z11, z15, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, + 192))); + _mm512_storeu_si512((void*)WC_PW(rcx, 192), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 448))); + _mm512_storeu_si512((void*)WC_PW(rcx, 448), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 704))); + _mm512_storeu_si512((void*)WC_PW(rcx, 704), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 960))); + _mm512_storeu_si512((void*)WC_PW(rcx, 960), z23); + z12 = _mm512_loadu_si512((const void*)WC_PR(r11, 768)); + z12 = _mm512_add_epi32(z12, _mm512_loadu_si512((const void*)WC_PR(rax, + 320))); + _mm512_storeu_si512((void*)WC_PW(r11, 768), z12); + z0 = _mm512_loadu_si512((const void*)WC_PR(r11, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r11, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r11, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r11, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r11, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(r11, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(r11, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r11, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(r11, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(r11, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(r11, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(r11, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(r11, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(r11, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(r11, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(r11, 960)); + rdx = (word64)(rdx + 0x400); + rcx = (word64)(rcx + 0x400); + r8 = (word32)((word32)r8 - 0x400); + if (((word32)r8) == (0)) { + goto L_cp512_epi; + } +L_cp512_start: + z16 = _mm512_loadu_si512((const void*)WC_PR(rsi, 752)); + z17 = _mm512_loadu_si512((const void*)WC_PR(rsi, 816)); + z18 = _mm512_loadu_si512((const void*)WC_PR(rsi, 880)); + z19 = _mm512_set1_epi64((long long)WC_L64(rsi, 720)); + z20 = _mm512_set1_epi64((long long)WC_L64(rsi, 728)); + z21 = _mm512_set1_epi64((long long)WC_L64(rsi, 736)); + z30 = _mm512_slli_epi64(z20, 2); + z22 = _mm512_slli_epi64(z20, 4); + z22 = _mm512_add_epi64(z22, z30); + z30 = _mm512_slli_epi64(z21, 2); + z23 = _mm512_slli_epi64(z21, 4); + z23 = _mm512_add_epi64(z23, z30); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -1024)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -960)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -896)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -832)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -768)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -704)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -640)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -576)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -512)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -448)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -384)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -320)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -256)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -192)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -128)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -64)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(r11, 0))); + z1 = _mm512_add_epi32(z1, _mm512_loadu_si512((const void*)WC_PR(r11, 64))); + z2 = _mm512_add_epi32(z2, _mm512_loadu_si512((const void*)WC_PR(r11, 128))); + z3 = _mm512_add_epi32(z3, _mm512_loadu_si512((const void*)WC_PR(r11, 192))); + z4 = _mm512_add_epi32(z4, _mm512_loadu_si512((const void*)WC_PR(r11, 256))); + z5 = _mm512_add_epi32(z5, _mm512_loadu_si512((const void*)WC_PR(r11, 320))); + z6 = _mm512_add_epi32(z6, _mm512_loadu_si512((const void*)WC_PR(r11, 384))); + z7 = _mm512_add_epi32(z7, _mm512_loadu_si512((const void*)WC_PR(r11, 448))); + z8 = _mm512_add_epi32(z8, _mm512_loadu_si512((const void*)WC_PR(r11, 512))); + z9 = _mm512_add_epi32(z9, _mm512_loadu_si512((const void*)WC_PR(r11, 576))); + z10 = _mm512_add_epi32(z10, _mm512_loadu_si512((const void*)WC_PR(r11, + 640))); + z11 = _mm512_add_epi32(z11, _mm512_loadu_si512((const void*)WC_PR(r11, + 704))); + z12 = _mm512_add_epi32(z12, _mm512_loadu_si512((const void*)WC_PR(r11, + 768))); + z13 = _mm512_add_epi32(z13, _mm512_loadu_si512((const void*)WC_PR(r11, + 832))); + z14 = _mm512_add_epi32(z14, _mm512_loadu_si512((const void*)WC_PR(r11, + 896))); + z15 = _mm512_add_epi32(z15, _mm512_loadu_si512((const void*)WC_PR(r11, + 960))); + _mm512_storeu_si512((void*)WC_PW(rsi, 752), z16); + _mm512_storeu_si512((void*)WC_PW(rsi, 816), z17); + _mm512_storeu_si512((void*)WC_PW(rsi, 880), z18); + z16 = _mm512_unpacklo_epi32(z0, z1); + z17 = _mm512_unpackhi_epi32(z0, z1); + z18 = _mm512_unpacklo_epi32(z2, z3); + z19 = _mm512_unpackhi_epi32(z2, z3); + z20 = _mm512_unpacklo_epi32(z4, z5); + z21 = _mm512_unpackhi_epi32(z4, z5); + z22 = _mm512_unpacklo_epi32(z6, z7); + z23 = _mm512_unpackhi_epi32(z6, z7); + z24 = _mm512_unpacklo_epi32(z8, z9); + z25 = _mm512_unpackhi_epi32(z8, z9); + z26 = _mm512_unpacklo_epi32(z10, z11); + z27 = _mm512_unpackhi_epi32(z10, z11); + z28 = _mm512_unpacklo_epi32(z12, z13); + z29 = _mm512_unpackhi_epi32(z12, z13); + z30 = _mm512_unpacklo_epi32(z14, z15); + z31 = _mm512_unpackhi_epi32(z14, z15); + z0 = _mm512_unpacklo_epi64(z16, z18); + z1 = _mm512_unpackhi_epi64(z16, z18); + z2 = _mm512_unpacklo_epi64(z17, z19); + z3 = _mm512_unpackhi_epi64(z17, z19); + z4 = _mm512_unpacklo_epi64(z20, z22); + z5 = _mm512_unpackhi_epi64(z20, z22); + z6 = _mm512_unpacklo_epi64(z21, z23); + z7 = _mm512_unpackhi_epi64(z21, z23); + z8 = _mm512_unpacklo_epi64(z24, z26); + z9 = _mm512_unpackhi_epi64(z24, z26); + z10 = _mm512_unpacklo_epi64(z25, z27); + z11 = _mm512_unpackhi_epi64(z25, z27); + z12 = _mm512_unpacklo_epi64(z28, z30); + z13 = _mm512_unpackhi_epi64(z28, z30); + z14 = _mm512_unpacklo_epi64(z29, z31); + z15 = _mm512_unpackhi_epi64(z29, z31); + z16 = _mm512_shuffle_i32x4(z0, z4, 0x44); + z17 = _mm512_shuffle_i32x4(z0, z4, 0xee); + z18 = _mm512_shuffle_i32x4(z8, z12, 0x44); + z19 = _mm512_shuffle_i32x4(z8, z12, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, 0))); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 256))); + _mm512_storeu_si512((void*)WC_PW(rcx, 256), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 512))); + _mm512_storeu_si512((void*)WC_PW(rcx, 512), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 768))); + _mm512_storeu_si512((void*)WC_PW(rcx, 768), z23); + z16 = _mm512_shuffle_i32x4(z1, z5, 0x44); + z17 = _mm512_shuffle_i32x4(z1, z5, 0xee); + z18 = _mm512_shuffle_i32x4(z9, z13, 0x44); + z19 = _mm512_shuffle_i32x4(z9, z13, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, + 64))); + _mm512_storeu_si512((void*)WC_PW(rcx, 64), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 320))); + _mm512_storeu_si512((void*)WC_PW(rcx, 320), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 576))); + _mm512_storeu_si512((void*)WC_PW(rcx, 576), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 832))); + _mm512_storeu_si512((void*)WC_PW(rcx, 832), z23); + z16 = _mm512_shuffle_i32x4(z2, z6, 0x44); + z17 = _mm512_shuffle_i32x4(z2, z6, 0xee); + z18 = _mm512_shuffle_i32x4(z10, z14, 0x44); + z19 = _mm512_shuffle_i32x4(z10, z14, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, + 128))); + _mm512_storeu_si512((void*)WC_PW(rcx, 128), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 384))); + _mm512_storeu_si512((void*)WC_PW(rcx, 384), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 640))); + _mm512_storeu_si512((void*)WC_PW(rcx, 640), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 896))); + _mm512_storeu_si512((void*)WC_PW(rcx, 896), z23); + z16 = _mm512_shuffle_i32x4(z3, z7, 0x44); + z17 = _mm512_shuffle_i32x4(z3, z7, 0xee); + z18 = _mm512_shuffle_i32x4(z11, z15, 0x44); + z19 = _mm512_shuffle_i32x4(z11, z15, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, + 192))); + _mm512_storeu_si512((void*)WC_PW(rcx, 192), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 448))); + _mm512_storeu_si512((void*)WC_PW(rcx, 448), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 704))); + _mm512_storeu_si512((void*)WC_PW(rcx, 704), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 960))); + _mm512_storeu_si512((void*)WC_PW(rcx, 960), z23); + z12 = _mm512_loadu_si512((const void*)WC_PR(r11, 768)); + z12 = _mm512_add_epi32(z12, _mm512_loadu_si512((const void*)WC_PR(rax, + 320))); + _mm512_storeu_si512((void*)WC_PW(r11, 768), z12); + z0 = _mm512_loadu_si512((const void*)WC_PR(r11, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r11, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r11, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r11, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r11, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(r11, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(r11, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r11, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(r11, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(r11, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(r11, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(r11, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(r11, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(r11, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(r11, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(r11, 960)); + rdx = (word64)(rdx + 0x400); + rcx = (word64)(rcx + 0x400); + r8 = (word32)((word32)r8 - 0x400); + if (((word32)r8) != (0)) { + goto L_cp512_start; + } +L_cp512_epi: + z16 = _mm512_loadu_si512((const void*)WC_PR(rsi, 752)); + z17 = _mm512_loadu_si512((const void*)WC_PR(rsi, 816)); + z18 = _mm512_loadu_si512((const void*)WC_PR(rsi, 880)); + z19 = _mm512_set1_epi64((long long)WC_L64(rsi, 720)); + z20 = _mm512_set1_epi64((long long)WC_L64(rsi, 728)); + z21 = _mm512_set1_epi64((long long)WC_L64(rsi, 736)); + z30 = _mm512_slli_epi64(z20, 2); + z22 = _mm512_slli_epi64(z20, 4); + z22 = _mm512_add_epi64(z22, z30); + z30 = _mm512_slli_epi64(z21, 2); + z23 = _mm512_slli_epi64(z21, 4); + z23 = _mm512_add_epi64(z23, z30); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -1024)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -960)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -896)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -832)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -768)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -704)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -640)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -576)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -512)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -448)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -384)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -320)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -256)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -192)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z27 = _mm512_loadu_si512((const void*)WC_PR(rcx, -128)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rcx, -64)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsi, 224)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsi, 288)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsi, 624)); + z27 = _mm512_loadu_si512((const void*)WC_PR(rsi, 688)); + z28 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z28 = _mm512_permutex2var_epi64(z26, z28, z27); + z29 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z29 = _mm512_permutex2var_epi64(z24, z29, z25); + z19 = _mm512_inserti64x4(z28, _mm512_castsi512_si256(z29), 1); + z28 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z28 = _mm512_permutex2var_epi64(z26, z28, z27); + z29 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z29 = _mm512_permutex2var_epi64(z24, z29, z25); + z20 = _mm512_inserti64x4(z28, _mm512_castsi512_si256(z29), 1); + z28 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z28 = _mm512_permutex2var_epi64(z26, z28, z27); + z29 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z29 = _mm512_permutex2var_epi64(z24, z29, z25); + z21 = _mm512_inserti64x4(z28, _mm512_castsi512_si256(z29), 1); + z30 = _mm512_slli_epi64(z20, 2); + z22 = _mm512_slli_epi64(z20, 4); + z22 = _mm512_add_epi64(z22, z30); + z30 = _mm512_slli_epi64(z21, 2); + z23 = _mm512_slli_epi64(z21, 4); + z23 = _mm512_add_epi64(z23, z30); + z24 = _mm512_setzero_si512(); + z25 = _mm512_setzero_si512(); + z26 = _mm512_setzero_si512(); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z24 = _mm512_shuffle_i64x2(z16, z16, 0x4e); + z25 = _mm512_shuffle_i64x2(z17, z17, 0x4e); + z26 = _mm512_shuffle_i64x2(z18, z18, 0x4e); + z16 = _mm512_add_epi64(z24, z16); + z17 = _mm512_add_epi64(z25, z17); + z18 = _mm512_add_epi64(z26, z18); + z24 = _mm512_bsrli_epi128(z16, 8); + z25 = _mm512_bsrli_epi128(z17, 8); + z26 = _mm512_bsrli_epi128(z18, 8); + z16 = _mm512_add_epi64(z24, z16); + z17 = _mm512_add_epi64(z25, z17); + z18 = _mm512_add_epi64(z26, z18); + z24 = _mm512_permutex_epi64(z16, 2); + z25 = _mm512_permutex_epi64(z17, 2); + z26 = _mm512_permutex_epi64(z18, 2); + z16 = _mm512_add_epi64(z24, z16); + z17 = _mm512_add_epi64(z25, z17); + z18 = _mm512_add_epi64(z26, z18); + r12 = (word64)((word64)_mm_cvtsi128_si64(_mm512_castsi512_si128(z16))); + r13 = (word64)((word64)_mm_cvtsi128_si64(_mm512_castsi512_si128(z17))); + r14 = (word64)((word64)_mm_cvtsi128_si64(_mm512_castsi512_si128(z18))); + r10 = (word64)(r13); + r10 = (word64)(r10 >> 20); + r11 = (word64)(r14); + r11 = (word64)(r11 >> 40); + rax = (word64)(r12); + r12 = (word64)(r13); + r12 = (word64)(r12 << 44); + cf = _addcarry_u64(0, rax, r12, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r12 = (word64)(r14); + r12 = (word64)(r12 << 24); + cf = _addcarry_u64(0, r10, r12, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + r9 = (word64)(r11); + r11 = (word64)(r11 & 3); + r9 = (word64)(r9 >> 2); + r9 = (word64)(r9 + r9 * 4); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + r12 = (word64)(WC_L64(rsi, 24)); + WC_S64(rsi, 64) = (word64)(r12); + r12 = (word64)(WC_L64(rsi, 32)); + WC_S64(rsi, 72) = (word64)(r12); + r12 = (word64)(WC_L64(rsi, 40)); + WC_S64(rsi, 80) = (word64)(r12); + WC_S64(rsi, 24) = (word64)(rax); + WC_S64(rsi, 32) = (word64)(r10); + WC_S64(rsi, 40) = (word64)(r11); + WC_S32(rdi, 48) = (word32)_mm_cvtsi128_si32(_mm512_castsi512_si128(z12)); +} + +WC_X64I_TARGET("avx512f,avx512bw,avx512ifma") +WOLFSSL_LOCAL void chacha20_poly1305_ifma_decrypt(ChaCha* chacha, + Poly1305* poly, const byte* m, byte* c, word32 bytes) +{ + word64 rdi, rsi, rdx, rcx, r8, rsp, rax, r10, r9, r11, r12 = 0, r13 = 0, + r14 = 0; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(), + z22 = _mm512_setzero_si512(), z23 = _mm512_setzero_si512(), + z24 = _mm512_setzero_si512(), z25 = _mm512_setzero_si512(), + z26 = _mm512_setzero_si512(), z27 = _mm512_setzero_si512(), + z28 = _mm512_setzero_si512(), z29 = _mm512_setzero_si512(), + z30 = _mm512_setzero_si512(), z31 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[152]; + unsigned char cf; + + rdi = (word64)(size_t)chacha; + rsi = (word64)(size_t)poly; + rdx = (word64)(size_t)m; + rcx = (word64)(size_t)c; + r8 = (word64)(word32)bytes; + + rsp = (word64)(size_t)stk + 1216; + rsp = (word64)(rsp - 1200); + rax = (word64)((word64)(size_t)L_cp512_pc); + r10 = (word64)((word64)(size_t)L_cp512_rx); + r9 = (word64)((word64)(size_t)L_cp512_add); + r11 = (word64)(rsp + 32); + r11 = (word64)(r11 + 0x3f); + r11 = (word64)(r11 & -64); + z16 = _mm512_setzero_si512(); + z17 = _mm512_setzero_si512(); + z18 = _mm512_setzero_si512(); + _mm512_storeu_si512((void*)WC_PW(rsi, 752), z16); + _mm512_storeu_si512((void*)WC_PW(rsi, 816), z17); + _mm512_storeu_si512((void*)WC_PW(rsi, 880), z18); + z0 = _mm512_set1_epi32((int)WC_L32(rdi, 0)); + z1 = _mm512_set1_epi32((int)WC_L32(rdi, 4)); + z2 = _mm512_set1_epi32((int)WC_L32(rdi, 8)); + z3 = _mm512_set1_epi32((int)WC_L32(rdi, 12)); + z4 = _mm512_set1_epi32((int)WC_L32(rdi, 16)); + z5 = _mm512_set1_epi32((int)WC_L32(rdi, 20)); + z6 = _mm512_set1_epi32((int)WC_L32(rdi, 24)); + z7 = _mm512_set1_epi32((int)WC_L32(rdi, 28)); + z8 = _mm512_set1_epi32((int)WC_L32(rdi, 32)); + z9 = _mm512_set1_epi32((int)WC_L32(rdi, 36)); + z10 = _mm512_set1_epi32((int)WC_L32(rdi, 40)); + z11 = _mm512_set1_epi32((int)WC_L32(rdi, 44)); + z12 = _mm512_set1_epi32((int)WC_L32(rdi, 48)); + z13 = _mm512_set1_epi32((int)WC_L32(rdi, 52)); + z14 = _mm512_set1_epi32((int)WC_L32(rdi, 56)); + z15 = _mm512_set1_epi32((int)WC_L32(rdi, 60)); + z12 = _mm512_add_epi32(z12, _mm512_loadu_si512((const void*)WC_PR(r9, 0))); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z1); + _mm512_storeu_si512((void*)WC_PW(r11, 128), z2); + _mm512_storeu_si512((void*)WC_PW(r11, 192), z3); + _mm512_storeu_si512((void*)WC_PW(r11, 256), z4); + _mm512_storeu_si512((void*)WC_PW(r11, 320), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 384), z6); + _mm512_storeu_si512((void*)WC_PW(r11, 448), z7); + _mm512_storeu_si512((void*)WC_PW(r11, 512), z8); + _mm512_storeu_si512((void*)WC_PW(r11, 576), z9); + _mm512_storeu_si512((void*)WC_PW(r11, 640), z10); + _mm512_storeu_si512((void*)WC_PW(r11, 704), z11); + _mm512_storeu_si512((void*)WC_PW(r11, 768), z12); + _mm512_storeu_si512((void*)WC_PW(r11, 832), z13); + _mm512_storeu_si512((void*)WC_PW(r11, 896), z14); + _mm512_storeu_si512((void*)WC_PW(r11, 960), z15); +L_cp512_dstart: + z16 = _mm512_loadu_si512((const void*)WC_PR(rsi, 752)); + z17 = _mm512_loadu_si512((const void*)WC_PR(rsi, 816)); + z18 = _mm512_loadu_si512((const void*)WC_PR(rsi, 880)); + z19 = _mm512_set1_epi64((long long)WC_L64(rsi, 720)); + z20 = _mm512_set1_epi64((long long)WC_L64(rsi, 728)); + z21 = _mm512_set1_epi64((long long)WC_L64(rsi, 736)); + z30 = _mm512_slli_epi64(z20, 2); + z22 = _mm512_slli_epi64(z20, 4); + z22 = _mm512_add_epi64(z22, z30); + z30 = _mm512_slli_epi64(z21, 2); + z23 = _mm512_slli_epi64(z21, 4); + z23 = _mm512_add_epi64(z23, z30); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z27 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z28 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z29 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z29 = _mm512_permutex2var_epi64(z27, z29, z28); + z30 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z30 = _mm512_permutex2var_epi64(z27, z30, z28); + z24 = _mm512_and_si512(z29, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z27 = _mm512_srli_epi64(z29, 44); + z28 = _mm512_and_si512(z30, _mm512_loadu_si512((const void*)WC_PR(rax, + 64))); + z28 = _mm512_slli_epi64(z28, 20); + z25 = _mm512_or_si512(z28, z27); + z26 = _mm512_srli_epi64(z30, 24); + z26 = _mm512_or_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, + 128))); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z15 = _mm512_rol_epi32(z15, 16); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 12); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z0 = _mm512_add_epi32(z0, z4); + z1 = _mm512_add_epi32(z1, z5); + z2 = _mm512_add_epi32(z2, z6); + z3 = _mm512_add_epi32(z3, z7); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z1); + z14 = _mm512_xor_si512(z14, z2); + z15 = _mm512_xor_si512(z15, z3); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z15 = _mm512_rol_epi32(z15, 8); + z8 = _mm512_add_epi32(z8, z12); + z9 = _mm512_add_epi32(z9, z13); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z15); + z4 = _mm512_xor_si512(z4, z8); + z5 = _mm512_xor_si512(z5, z9); + z6 = _mm512_xor_si512(z6, z10); + z7 = _mm512_xor_si512(z7, z11); + z4 = _mm512_rol_epi32(z4, 7); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 16); + z12 = _mm512_rol_epi32(z12, 16); + z13 = _mm512_rol_epi32(z13, 16); + z14 = _mm512_rol_epi32(z14, 16); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 12); + z6 = _mm512_rol_epi32(z6, 12); + z7 = _mm512_rol_epi32(z7, 12); + z4 = _mm512_rol_epi32(z4, 12); + z0 = _mm512_add_epi32(z0, z5); + z1 = _mm512_add_epi32(z1, z6); + z2 = _mm512_add_epi32(z2, z7); + z3 = _mm512_add_epi32(z3, z4); + z15 = _mm512_xor_si512(z15, z0); + z12 = _mm512_xor_si512(z12, z1); + z13 = _mm512_xor_si512(z13, z2); + z14 = _mm512_xor_si512(z14, z3); + z15 = _mm512_rol_epi32(z15, 8); + z12 = _mm512_rol_epi32(z12, 8); + z13 = _mm512_rol_epi32(z13, 8); + z14 = _mm512_rol_epi32(z14, 8); + z10 = _mm512_add_epi32(z10, z15); + z11 = _mm512_add_epi32(z11, z12); + z8 = _mm512_add_epi32(z8, z13); + z9 = _mm512_add_epi32(z9, z14); + z5 = _mm512_xor_si512(z5, z10); + z6 = _mm512_xor_si512(z6, z11); + z7 = _mm512_xor_si512(z7, z8); + z4 = _mm512_xor_si512(z4, z9); + z5 = _mm512_rol_epi32(z5, 7); + z6 = _mm512_rol_epi32(z6, 7); + z7 = _mm512_rol_epi32(z7, 7); + z4 = _mm512_rol_epi32(z4, 7); + z0 = _mm512_add_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(r11, 0))); + z1 = _mm512_add_epi32(z1, _mm512_loadu_si512((const void*)WC_PR(r11, 64))); + z2 = _mm512_add_epi32(z2, _mm512_loadu_si512((const void*)WC_PR(r11, 128))); + z3 = _mm512_add_epi32(z3, _mm512_loadu_si512((const void*)WC_PR(r11, 192))); + z4 = _mm512_add_epi32(z4, _mm512_loadu_si512((const void*)WC_PR(r11, 256))); + z5 = _mm512_add_epi32(z5, _mm512_loadu_si512((const void*)WC_PR(r11, 320))); + z6 = _mm512_add_epi32(z6, _mm512_loadu_si512((const void*)WC_PR(r11, 384))); + z7 = _mm512_add_epi32(z7, _mm512_loadu_si512((const void*)WC_PR(r11, 448))); + z8 = _mm512_add_epi32(z8, _mm512_loadu_si512((const void*)WC_PR(r11, 512))); + z9 = _mm512_add_epi32(z9, _mm512_loadu_si512((const void*)WC_PR(r11, 576))); + z10 = _mm512_add_epi32(z10, _mm512_loadu_si512((const void*)WC_PR(r11, + 640))); + z11 = _mm512_add_epi32(z11, _mm512_loadu_si512((const void*)WC_PR(r11, + 704))); + z12 = _mm512_add_epi32(z12, _mm512_loadu_si512((const void*)WC_PR(r11, + 768))); + z13 = _mm512_add_epi32(z13, _mm512_loadu_si512((const void*)WC_PR(r11, + 832))); + z14 = _mm512_add_epi32(z14, _mm512_loadu_si512((const void*)WC_PR(r11, + 896))); + z15 = _mm512_add_epi32(z15, _mm512_loadu_si512((const void*)WC_PR(r11, + 960))); + _mm512_storeu_si512((void*)WC_PW(rsi, 752), z16); + _mm512_storeu_si512((void*)WC_PW(rsi, 816), z17); + _mm512_storeu_si512((void*)WC_PW(rsi, 880), z18); + z16 = _mm512_unpacklo_epi32(z0, z1); + z17 = _mm512_unpackhi_epi32(z0, z1); + z18 = _mm512_unpacklo_epi32(z2, z3); + z19 = _mm512_unpackhi_epi32(z2, z3); + z20 = _mm512_unpacklo_epi32(z4, z5); + z21 = _mm512_unpackhi_epi32(z4, z5); + z22 = _mm512_unpacklo_epi32(z6, z7); + z23 = _mm512_unpackhi_epi32(z6, z7); + z24 = _mm512_unpacklo_epi32(z8, z9); + z25 = _mm512_unpackhi_epi32(z8, z9); + z26 = _mm512_unpacklo_epi32(z10, z11); + z27 = _mm512_unpackhi_epi32(z10, z11); + z28 = _mm512_unpacklo_epi32(z12, z13); + z29 = _mm512_unpackhi_epi32(z12, z13); + z30 = _mm512_unpacklo_epi32(z14, z15); + z31 = _mm512_unpackhi_epi32(z14, z15); + z0 = _mm512_unpacklo_epi64(z16, z18); + z1 = _mm512_unpackhi_epi64(z16, z18); + z2 = _mm512_unpacklo_epi64(z17, z19); + z3 = _mm512_unpackhi_epi64(z17, z19); + z4 = _mm512_unpacklo_epi64(z20, z22); + z5 = _mm512_unpackhi_epi64(z20, z22); + z6 = _mm512_unpacklo_epi64(z21, z23); + z7 = _mm512_unpackhi_epi64(z21, z23); + z8 = _mm512_unpacklo_epi64(z24, z26); + z9 = _mm512_unpackhi_epi64(z24, z26); + z10 = _mm512_unpacklo_epi64(z25, z27); + z11 = _mm512_unpackhi_epi64(z25, z27); + z12 = _mm512_unpacklo_epi64(z28, z30); + z13 = _mm512_unpackhi_epi64(z28, z30); + z14 = _mm512_unpacklo_epi64(z29, z31); + z15 = _mm512_unpackhi_epi64(z29, z31); + z16 = _mm512_shuffle_i32x4(z0, z4, 0x44); + z17 = _mm512_shuffle_i32x4(z0, z4, 0xee); + z18 = _mm512_shuffle_i32x4(z8, z12, 0x44); + z19 = _mm512_shuffle_i32x4(z8, z12, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, 0))); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 256))); + _mm512_storeu_si512((void*)WC_PW(rcx, 256), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 512))); + _mm512_storeu_si512((void*)WC_PW(rcx, 512), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 768))); + _mm512_storeu_si512((void*)WC_PW(rcx, 768), z23); + z16 = _mm512_shuffle_i32x4(z1, z5, 0x44); + z17 = _mm512_shuffle_i32x4(z1, z5, 0xee); + z18 = _mm512_shuffle_i32x4(z9, z13, 0x44); + z19 = _mm512_shuffle_i32x4(z9, z13, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, + 64))); + _mm512_storeu_si512((void*)WC_PW(rcx, 64), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 320))); + _mm512_storeu_si512((void*)WC_PW(rcx, 320), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 576))); + _mm512_storeu_si512((void*)WC_PW(rcx, 576), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 832))); + _mm512_storeu_si512((void*)WC_PW(rcx, 832), z23); + z16 = _mm512_shuffle_i32x4(z2, z6, 0x44); + z17 = _mm512_shuffle_i32x4(z2, z6, 0xee); + z18 = _mm512_shuffle_i32x4(z10, z14, 0x44); + z19 = _mm512_shuffle_i32x4(z10, z14, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, + 128))); + _mm512_storeu_si512((void*)WC_PW(rcx, 128), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 384))); + _mm512_storeu_si512((void*)WC_PW(rcx, 384), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 640))); + _mm512_storeu_si512((void*)WC_PW(rcx, 640), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 896))); + _mm512_storeu_si512((void*)WC_PW(rcx, 896), z23); + z16 = _mm512_shuffle_i32x4(z3, z7, 0x44); + z17 = _mm512_shuffle_i32x4(z3, z7, 0xee); + z18 = _mm512_shuffle_i32x4(z11, z15, 0x44); + z19 = _mm512_shuffle_i32x4(z11, z15, 0xee); + z20 = _mm512_shuffle_i32x4(z16, z18, 0x88); + z21 = _mm512_shuffle_i32x4(z16, z18, 0xdd); + z22 = _mm512_shuffle_i32x4(z17, z19, 0x88); + z23 = _mm512_shuffle_i32x4(z17, z19, 0xdd); + z20 = _mm512_xor_si512(z20, _mm512_loadu_si512((const void*)WC_PR(rdx, + 192))); + _mm512_storeu_si512((void*)WC_PW(rcx, 192), z20); + z21 = _mm512_xor_si512(z21, _mm512_loadu_si512((const void*)WC_PR(rdx, + 448))); + _mm512_storeu_si512((void*)WC_PW(rcx, 448), z21); + z22 = _mm512_xor_si512(z22, _mm512_loadu_si512((const void*)WC_PR(rdx, + 704))); + _mm512_storeu_si512((void*)WC_PW(rcx, 704), z22); + z23 = _mm512_xor_si512(z23, _mm512_loadu_si512((const void*)WC_PR(rdx, + 960))); + _mm512_storeu_si512((void*)WC_PW(rcx, 960), z23); + z12 = _mm512_loadu_si512((const void*)WC_PR(r11, 768)); + z12 = _mm512_add_epi32(z12, _mm512_loadu_si512((const void*)WC_PR(rax, + 320))); + _mm512_storeu_si512((void*)WC_PW(r11, 768), z12); + z0 = _mm512_loadu_si512((const void*)WC_PR(r11, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r11, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r11, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r11, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r11, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(r11, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(r11, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r11, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(r11, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(r11, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(r11, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(r11, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(r11, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(r11, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(r11, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(r11, 960)); + rdx = (word64)(rdx + 0x400); + rcx = (word64)(rcx + 0x400); + r8 = (word32)((word32)r8 - 0x400); + if (((word32)r8) != (0)) { + goto L_cp512_dstart; + } + z16 = _mm512_loadu_si512((const void*)WC_PR(rsi, 752)); + z17 = _mm512_loadu_si512((const void*)WC_PR(rsi, 816)); + z18 = _mm512_loadu_si512((const void*)WC_PR(rsi, 880)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rsi, 224)); + z25 = _mm512_loadu_si512((const void*)WC_PR(rsi, 288)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsi, 624)); + z27 = _mm512_loadu_si512((const void*)WC_PR(rsi, 688)); + z28 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z28 = _mm512_permutex2var_epi64(z26, z28, z27); + z29 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z29 = _mm512_permutex2var_epi64(z24, z29, z25); + z19 = _mm512_inserti64x4(z28, _mm512_castsi512_si256(z29), 1); + z28 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z28 = _mm512_permutex2var_epi64(z26, z28, z27); + z29 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z29 = _mm512_permutex2var_epi64(z24, z29, z25); + z20 = _mm512_inserti64x4(z28, _mm512_castsi512_si256(z29), 1); + z28 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z28 = _mm512_permutex2var_epi64(z26, z28, z27); + z29 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z29 = _mm512_permutex2var_epi64(z24, z29, z25); + z21 = _mm512_inserti64x4(z28, _mm512_castsi512_si256(z29), 1); + z30 = _mm512_slli_epi64(z20, 2); + z22 = _mm512_slli_epi64(z20, 4); + z22 = _mm512_add_epi64(z22, z30); + z30 = _mm512_slli_epi64(z21, 2); + z23 = _mm512_slli_epi64(z21, 4); + z23 = _mm512_add_epi64(z23, z30); + z24 = _mm512_setzero_si512(); + z25 = _mm512_setzero_si512(); + z26 = _mm512_setzero_si512(); + z27 = _mm512_setzero_si512(); + z28 = _mm512_setzero_si512(); + z29 = _mm512_setzero_si512(); + z24 = _mm512_madd52lo_epu64(z24, z16, z19); + z24 = _mm512_madd52lo_epu64(z24, z17, z23); + z24 = _mm512_madd52lo_epu64(z24, z18, z22); + z27 = _mm512_madd52hi_epu64(z27, z16, z19); + z27 = _mm512_madd52hi_epu64(z27, z17, z23); + z27 = _mm512_madd52hi_epu64(z27, z18, z22); + z25 = _mm512_madd52lo_epu64(z25, z16, z20); + z25 = _mm512_madd52lo_epu64(z25, z17, z19); + z25 = _mm512_madd52lo_epu64(z25, z18, z23); + z28 = _mm512_madd52hi_epu64(z28, z16, z20); + z28 = _mm512_madd52hi_epu64(z28, z17, z19); + z28 = _mm512_madd52hi_epu64(z28, z18, z23); + z26 = _mm512_madd52lo_epu64(z26, z16, z21); + z26 = _mm512_madd52lo_epu64(z26, z17, z20); + z26 = _mm512_madd52lo_epu64(z26, z18, z19); + z29 = _mm512_madd52hi_epu64(z29, z16, z21); + z29 = _mm512_madd52hi_epu64(z29, z17, z20); + z29 = _mm512_madd52hi_epu64(z29, z18, z19); + z30 = _mm512_slli_epi64(z27, 8); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_slli_epi64(z28, 8); + z26 = _mm512_add_epi64(z30, z26); + z31 = _mm512_slli_epi64(z29, 8); + z30 = _mm512_slli_epi64(z31, 2); + z31 = _mm512_slli_epi64(z31, 4); + z31 = _mm512_add_epi64(z31, z30); + z24 = _mm512_add_epi64(z31, z24); + z30 = _mm512_srli_epi64(z24, 44); + z16 = _mm512_and_si512(z24, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z25 = _mm512_add_epi64(z30, z25); + z30 = _mm512_srli_epi64(z25, 44); + z17 = _mm512_and_si512(z25, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z26 = _mm512_add_epi64(z30, z26); + z30 = _mm512_srli_epi64(z26, 44); + z18 = _mm512_and_si512(z26, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + z31 = _mm512_slli_epi64(z30, 2); + z30 = _mm512_slli_epi64(z30, 4); + z30 = _mm512_add_epi64(z30, z31); + z16 = _mm512_add_epi64(z30, z16); + z24 = _mm512_shuffle_i64x2(z16, z16, 0x4e); + z25 = _mm512_shuffle_i64x2(z17, z17, 0x4e); + z26 = _mm512_shuffle_i64x2(z18, z18, 0x4e); + z16 = _mm512_add_epi64(z24, z16); + z17 = _mm512_add_epi64(z25, z17); + z18 = _mm512_add_epi64(z26, z18); + z24 = _mm512_bsrli_epi128(z16, 8); + z25 = _mm512_bsrli_epi128(z17, 8); + z26 = _mm512_bsrli_epi128(z18, 8); + z16 = _mm512_add_epi64(z24, z16); + z17 = _mm512_add_epi64(z25, z17); + z18 = _mm512_add_epi64(z26, z18); + z24 = _mm512_permutex_epi64(z16, 2); + z25 = _mm512_permutex_epi64(z17, 2); + z26 = _mm512_permutex_epi64(z18, 2); + z16 = _mm512_add_epi64(z24, z16); + z17 = _mm512_add_epi64(z25, z17); + z18 = _mm512_add_epi64(z26, z18); + r12 = (word64)((word64)_mm_cvtsi128_si64(_mm512_castsi512_si128(z16))); + r13 = (word64)((word64)_mm_cvtsi128_si64(_mm512_castsi512_si128(z17))); + r14 = (word64)((word64)_mm_cvtsi128_si64(_mm512_castsi512_si128(z18))); + r10 = (word64)(r13); + r10 = (word64)(r10 >> 20); + r11 = (word64)(r14); + r11 = (word64)(r11 >> 40); + rax = (word64)(r12); + r12 = (word64)(r13); + r12 = (word64)(r12 << 44); + cf = _addcarry_u64(0, rax, r12, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r12 = (word64)(r14); + r12 = (word64)(r12 << 24); + cf = _addcarry_u64(0, r10, r12, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + r9 = (word64)(r11); + r11 = (word64)(r11 & 3); + r9 = (word64)(r9 >> 2); + r9 = (word64)(r9 + r9 * 4); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + r12 = (word64)(WC_L64(rsi, 24)); + WC_S64(rsi, 64) = (word64)(r12); + r12 = (word64)(WC_L64(rsi, 32)); + WC_S64(rsi, 72) = (word64)(r12); + r12 = (word64)(WC_L64(rsi, 40)); + WC_S64(rsi, 80) = (word64)(r12); + WC_S64(rsi, 24) = (word64)(rax); + WC_S64(rsi, 32) = (word64)(r10); + WC_S64(rsi, 40) = (word64)(r11); + WC_S32(rdi, 48) = (word32)_mm_cvtsi128_si32(_mm512_castsi512_si128(z12)); +} + +#endif /* HAVE_INTEL_AVX512 */ +#ifdef HAVE_INTEL_AVX2 +XALIGNED(32) static const word64 L_chacha20_poly1305_small_enc_rotl8[] + WC_X64I_UNUSED = { + 0x0605040702010003ULL, 0x0e0d0c0f0a09080bULL, + 0x0605040702010003ULL, 0x0e0d0c0f0a09080bULL, +}; + +XALIGNED(32) static const word64 L_chacha20_poly1305_small_enc_rotl16[] + WC_X64I_UNUSED = { + 0x0504070601000302ULL, 0x0d0c0f0e09080b0aULL, + 0x0504070601000302ULL, 0x0d0c0f0e09080b0aULL, +}; + +XALIGNED(32) static const word64 L_chacha20_poly1305_small_enc_ymm_inc[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void chacha20_poly1305_small_enc(ChaCha* chacha, Poly1305* poly, + const byte* m, byte* c, word32 mLen, const byte* aad, word32 aadLen, + byte* tag) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rsp, rax, r11, r12, r13, rbp = 0, + rbx = 0, r10 = 0, r14 = 0, r15 = 0; + __m128i x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(); + __m256i y0, y1, y2, y3, y8 = _mm256_setzero_si256(), y10, y11, y12, y13; + XALIGNED(32) WC_X64I_SLOT stk[24]; + word64 zf1; + word64 zf2; + unsigned char cf; + + rdi = (word64)(size_t)chacha; + rsi = (word64)(size_t)poly; + rdx = (word64)(size_t)m; + rcx = (word64)(size_t)c; + r8 = (word64)(word32)mLen; + r9 = (word64)(size_t)aad; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 192); + rax = (word32)((word32)aadLen); + WC_S64(rsp, 128) = (word64)(rax); + rax = (word64)(tag); + WC_S64(rsp, 152) = (word64)(rax); + WC_S64(rsp, 136) = (word64)(rcx); + rax = (word64)(r8); + WC_S64(rsp, 144) = (word64)(rax); + r11 = (word64)((word64)(size_t)L_chacha20_poly1305_small_enc_rotl8); + r12 = (word64)((word64)(size_t)L_chacha20_poly1305_small_enc_rotl16); + r13 = (word64)((word64)(size_t)L_chacha20_poly1305_small_enc_ymm_inc); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 0))); + y1 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + y2 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + y3 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + y3 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + y10 = y0; + y11 = y1; + y12 = y2; + y13 = y3; + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_c20p1305_small_enc_crypt2_start: + y0 = _mm256_add_epi32(y0, y1); + y3 = _mm256_xor_si256(y3, y0); + y3 = _mm256_shuffle_epi8(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y2 = _mm256_add_epi32(y2, y3); + y1 = _mm256_xor_si256(y1, y2); + y8 = _mm256_srli_epi32(y1, 20); + y1 = _mm256_slli_epi32(y1, 12); + y1 = _mm256_xor_si256(y1, y8); + y0 = _mm256_add_epi32(y0, y1); + y3 = _mm256_xor_si256(y3, y0); + y3 = _mm256_shuffle_epi8(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y2 = _mm256_add_epi32(y2, y3); + y1 = _mm256_xor_si256(y1, y2); + y8 = _mm256_srli_epi32(y1, 25); + y1 = _mm256_slli_epi32(y1, 7); + y1 = _mm256_xor_si256(y1, y8); + y1 = _mm256_shuffle_epi32(y1, 0x39); + y2 = _mm256_shuffle_epi32(y2, 0x4e); + y3 = _mm256_shuffle_epi32(y3, 0x93); + y0 = _mm256_add_epi32(y0, y1); + y3 = _mm256_xor_si256(y3, y0); + y3 = _mm256_shuffle_epi8(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y2 = _mm256_add_epi32(y2, y3); + y1 = _mm256_xor_si256(y1, y2); + y8 = _mm256_srli_epi32(y1, 20); + y1 = _mm256_slli_epi32(y1, 12); + y1 = _mm256_xor_si256(y1, y8); + y0 = _mm256_add_epi32(y0, y1); + y3 = _mm256_xor_si256(y3, y0); + y3 = _mm256_shuffle_epi8(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y2 = _mm256_add_epi32(y2, y3); + y1 = _mm256_xor_si256(y1, y2); + y8 = _mm256_srli_epi32(y1, 25); + y1 = _mm256_slli_epi32(y1, 7); + y1 = _mm256_xor_si256(y1, y8); + y1 = _mm256_shuffle_epi32(y1, 0x93); + y2 = _mm256_shuffle_epi32(y2, 0x4e); + y3 = _mm256_shuffle_epi32(y3, 0x39); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 1) & 0xff); + if (((byte)rcx) != (0)) { + goto L_c20p1305_small_enc_crypt2_start; + } + y0 = _mm256_add_epi32(y0, y10); + y1 = _mm256_add_epi32(y1, y11); + y2 = _mm256_add_epi32(y2, y12); + y3 = _mm256_add_epi32(y3, y13); + x4 = _mm256_extracti128_si256(y0, 1); + x5 = _mm256_extracti128_si256(y1, 1); + x6 = _mm256_extracti128_si256(y2, 1); + x7 = _mm256_extracti128_si256(y3, 1); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + rcx = (word64)(WC_L64(rsp, 136)); + rbp = (word64)(WC_L64(rsp, 144)); + rbx = (word64)(rsp + 32); +L_chacha20_poly1305_small_enc_x16: + if ((sword64)(rbp) < (sword64)(0x10)) { + goto L_chacha20_poly1305_small_enc_xtail; + } + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(rbx, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y8)); + rdx = (word64)(rdx + 0x10); + rcx = (word64)(rcx + 0x10); + rbx = (word64)(rbx + 0x10); + rbp = (word64)(rbp - 0x10); + goto L_chacha20_poly1305_small_enc_x16; +L_chacha20_poly1305_small_enc_xtail: + if (((rbp & rbp)) == (0)) { + goto L_chacha20_poly1305_small_enc_xdone; + } + rdi = (word64)(0); +L_chacha20_poly1305_small_enc_xbyte: + if ((sword64)(rdi) >= (sword64)(rbp)) { + goto L_chacha20_poly1305_small_enc_xdone; + } + rax = (word32)((word32)WC_L8(rdx, rdi)); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax ^ WC_L8(rbx, + rdi)) & 0xff); + WC_S8(rcx, rdi) = (byte)((byte)rax); + rdi = (word64)(rdi + 1); + goto L_chacha20_poly1305_small_enc_xbyte; +L_chacha20_poly1305_small_enc_xdone: + r10 = (word64)(rsi); + rax = (word64)(0xffffffc0fffffffULL); + rdx = (word64)(0xffffffc0ffffffcULL); + r11 = (word64)(WC_L64(rsp, 0)); + r11 = (word64)(r11 & rax); + r12 = (word64)(WC_L64(rsp, 8)); + r12 = (word64)(r12 & rdx); + rax = (word64)(WC_L64(rsp, 16)); + WC_S64(r10, 48) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 24)); + WC_S64(r10, 56) = (word64)(rax); + rax = (word64)(0); + WC_S64(r10, 352) = (word64)(rax); + WC_S64(r10, 408) = (word64)(rax); + WC_S64(r10, 360) = (word64)(r11); + WC_S64(r10, 416) = (word64)(r12); + rcx = (word64)(r11); + rsi = (word64)(r12); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 368) = (word64)(rcx); + WC_S64(r10, 424) = (word64)(rsi); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 376) = (word64)(rcx); + WC_S64(r10, 432) = (word64)(rsi); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 384) = (word64)(rcx); + WC_S64(r10, 440) = (word64)(rsi); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 392) = (word64)(rcx); + WC_S64(r10, 448) = (word64)(rsi); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 400) = (word64)(rcx); + WC_S64(r10, 456) = (word64)(rsi); + r13 = (word64)(0); + r14 = (word64)(0); + r15 = (word64)(0); + rbx = (word64)(r9); + rbp = (word64)(WC_L64(rsp, 128)); +L_c20p1305_small_aad_full: + if ((sword64)(rbp) < (sword64)(0x10)) { + goto L_c20p1305_small_aad_part; + } + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rbx = (word64)(rbx + 0x10); + rbp = (word64)(rbp - 0x10); + goto L_c20p1305_small_aad_full; +L_c20p1305_small_aad_part: + if (((rbp & rbp)) == (0)) { + goto L_c20p1305_small_aad_done; + } + rax = (word64)(0); + WC_S64(rsp, 96) = (word64)(rax); + WC_S64(rsp, 104) = (word64)(rax); + r9 = (word64)(rsp + 96); + if ((sword64)(rbp) < (sword64)(8)) { + goto L_c20p1305_small_aad_c4; + } + rax = (word64)(WC_L64(rbx, 0)); + WC_S64(r9, 0) = (word64)(rax); + rbx = (word64)(rbx + 8); + r9 = (word64)(r9 + 8); + rbp = (word64)(rbp - 8); +L_c20p1305_small_aad_c4: + if ((sword64)(rbp) < (sword64)(4)) { + goto L_c20p1305_small_aad_cby; + } + rax = (word32)(WC_L32(rbx, 0)); + WC_S32(r9, 0) = (word32)((word32)rax); + rbx = (word64)(rbx + 4); + r9 = (word64)(r9 + 4); + rbp = (word64)(rbp - 4); +L_c20p1305_small_aad_cby: + if (((rbp & rbp)) == (0)) { + goto L_c20p1305_small_aad_cpyd; + } +L_c20p1305_small_aad_cbyl: + rax = (word32)((word32)WC_L8(rbx, 0)); + WC_S8(r9, 0) = (byte)((byte)rax); + rbx = (word64)(rbx + 1); + r9 = (word64)(r9 + 1); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_c20p1305_small_aad_cbyl; + } +L_c20p1305_small_aad_cpyd: + rbx = (word64)(rsp + 96); + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); +L_c20p1305_small_aad_done: + rbx = (word64)(WC_L64(rsp, 136)); + rbp = (word64)(WC_L64(rsp, 144)); +L_c20p1305_small_ct_full: + if ((sword64)(rbp) < (sword64)(0x10)) { + goto L_c20p1305_small_ct_part; + } + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rbx = (word64)(rbx + 0x10); + rbp = (word64)(rbp - 0x10); + goto L_c20p1305_small_ct_full; +L_c20p1305_small_ct_part: + if (((rbp & rbp)) == (0)) { + goto L_c20p1305_small_ct_done; + } + rax = (word64)(0); + WC_S64(rsp, 96) = (word64)(rax); + WC_S64(rsp, 104) = (word64)(rax); + r9 = (word64)(rsp + 96); + if ((sword64)(rbp) < (sword64)(8)) { + goto L_c20p1305_small_ct_c4; + } + rax = (word64)(WC_L64(rbx, 0)); + WC_S64(r9, 0) = (word64)(rax); + rbx = (word64)(rbx + 8); + r9 = (word64)(r9 + 8); + rbp = (word64)(rbp - 8); +L_c20p1305_small_ct_c4: + if ((sword64)(rbp) < (sword64)(4)) { + goto L_c20p1305_small_ct_cby; + } + rax = (word32)(WC_L32(rbx, 0)); + WC_S32(r9, 0) = (word32)((word32)rax); + rbx = (word64)(rbx + 4); + r9 = (word64)(r9 + 4); + rbp = (word64)(rbp - 4); +L_c20p1305_small_ct_cby: + if (((rbp & rbp)) == (0)) { + goto L_c20p1305_small_ct_cpyd; + } +L_c20p1305_small_ct_cbyl: + rax = (word32)((word32)WC_L8(rbx, 0)); + WC_S8(r9, 0) = (byte)((byte)rax); + rbx = (word64)(rbx + 1); + r9 = (word64)(r9 + 1); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_c20p1305_small_ct_cbyl; + } +L_c20p1305_small_ct_cpyd: + rbx = (word64)(rsp + 96); + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); +L_c20p1305_small_ct_done: + rax = (word64)(WC_L64(rsp, 128)); + WC_S64(rsp, 112) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 144)); + WC_S64(rsp, 120) = (word64)(rax); + rbx = (word64)(rsp + 112); + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rbx = (word64)(WC_L64(rsp, 152)); + rcx = (word64)(r15); + r15 = (word64)(r15 & 3); + rcx = (word64)(rcx >> 2); + rcx = (word64)(rcx + rcx * 4); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rcx = (word64)(r13); + rsi = (word64)(r14); + rdi = (word64)(r15); + cf = _addcarry_u64(0, rcx, 5, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, 0, (unsigned long long*)&rdi); + zf1 = rdi; + zf2 = 4; + r13 = (rdi) == (4) ? rcx : r13; + r14 = (zf1) == (zf2) ? rsi : r14; + cf = _addcarry_u64(0, r13, WC_L64(r10, 48), (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, WC_L64(r10, 56), (unsigned long long*)&r14); + WC_S64(rbx, 0) = (word64)(r13); + WC_S64(rbx, 8) = (word64)(r14); + y0 = _mm256_zextsi128_si256(_mm_setzero_si128()); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), _mm256_castsi256_si128(y0)); + (void)aadLen; + (void)tag; +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL int chacha20_poly1305_small_dec(ChaCha* chacha, Poly1305* poly, + const byte* in, byte* out, word32 ctLen, const byte* aad, word32 aadLen, + const byte* tag) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rsp, rax, r11, r12, r13, r10 = 0, + r14 = 0, r15 = 0, rbx = 0, rbp = 0; + __m128i x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(); + __m256i y0, y1, y2, y3, y8 = _mm256_setzero_si256(), y10, y11, y12, y13; + XALIGNED(32) WC_X64I_SLOT stk[24]; + word64 zf1; + word64 zf2; + unsigned char cf; + + rdi = (word64)(size_t)chacha; + rsi = (word64)(size_t)poly; + rdx = (word64)(size_t)in; + rcx = (word64)(size_t)out; + r8 = (word64)(word32)ctLen; + r9 = (word64)(size_t)aad; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 168); + rax = (word32)((word32)aadLen); + WC_S64(rsp, 128) = (word64)(rax); + rax = (word64)(tag); + WC_S64(rsp, 144) = (word64)(rax); + rax = (word64)(r8); + WC_S64(rsp, 136) = (word64)(rax); + rax = (word64)(rdx); + WC_S64(rsp, 152) = (word64)(rax); + rax = (word64)(rcx); + WC_S64(rsp, 160) = (word64)(rax); + r11 = (word64)((word64)(size_t)L_chacha20_poly1305_small_enc_rotl8); + r12 = (word64)((word64)(size_t)L_chacha20_poly1305_small_enc_rotl16); + r13 = (word64)((word64)(size_t)L_chacha20_poly1305_small_enc_ymm_inc); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 0))); + y1 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 16))); + y2 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 32))); + y3 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdi, + 48))); + y3 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + y10 = y0; + y11 = y1; + y12 = y2; + y13 = y3; + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(0xa) & 0xff); +L_c20p1305_small_dec_crypt2_start: + y0 = _mm256_add_epi32(y0, y1); + y3 = _mm256_xor_si256(y3, y0); + y3 = _mm256_shuffle_epi8(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y2 = _mm256_add_epi32(y2, y3); + y1 = _mm256_xor_si256(y1, y2); + y8 = _mm256_srli_epi32(y1, 20); + y1 = _mm256_slli_epi32(y1, 12); + y1 = _mm256_xor_si256(y1, y8); + y0 = _mm256_add_epi32(y0, y1); + y3 = _mm256_xor_si256(y3, y0); + y3 = _mm256_shuffle_epi8(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y2 = _mm256_add_epi32(y2, y3); + y1 = _mm256_xor_si256(y1, y2); + y8 = _mm256_srli_epi32(y1, 25); + y1 = _mm256_slli_epi32(y1, 7); + y1 = _mm256_xor_si256(y1, y8); + y1 = _mm256_shuffle_epi32(y1, 0x39); + y2 = _mm256_shuffle_epi32(y2, 0x4e); + y3 = _mm256_shuffle_epi32(y3, 0x93); + y0 = _mm256_add_epi32(y0, y1); + y3 = _mm256_xor_si256(y3, y0); + y3 = _mm256_shuffle_epi8(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + 0))); + y2 = _mm256_add_epi32(y2, y3); + y1 = _mm256_xor_si256(y1, y2); + y8 = _mm256_srli_epi32(y1, 20); + y1 = _mm256_slli_epi32(y1, 12); + y1 = _mm256_xor_si256(y1, y8); + y0 = _mm256_add_epi32(y0, y1); + y3 = _mm256_xor_si256(y3, y0); + y3 = _mm256_shuffle_epi8(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + y2 = _mm256_add_epi32(y2, y3); + y1 = _mm256_xor_si256(y1, y2); + y8 = _mm256_srli_epi32(y1, 25); + y1 = _mm256_slli_epi32(y1, 7); + y1 = _mm256_xor_si256(y1, y8); + y1 = _mm256_shuffle_epi32(y1, 0x93); + y2 = _mm256_shuffle_epi32(y2, 0x4e); + y3 = _mm256_shuffle_epi32(y3, 0x39); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 1) & 0xff); + if (((byte)rcx) != (0)) { + goto L_c20p1305_small_dec_crypt2_start; + } + y0 = _mm256_add_epi32(y0, y10); + y1 = _mm256_add_epi32(y1, y11); + y2 = _mm256_add_epi32(y2, y12); + y3 = _mm256_add_epi32(y3, y13); + x4 = _mm256_extracti128_si256(y0, 1); + x5 = _mm256_extracti128_si256(y1, 1); + x6 = _mm256_extracti128_si256(y2, 1); + x7 = _mm256_extracti128_si256(y3, 1); + r10 = (word64)(rsi); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x5); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x7); + rax = (word64)(0xffffffc0fffffffULL); + rdx = (word64)(0xffffffc0ffffffcULL); + r11 = (word64)(WC_L64(rsp, 0)); + r11 = (word64)(r11 & rax); + r12 = (word64)(WC_L64(rsp, 8)); + r12 = (word64)(r12 & rdx); + rax = (word64)(WC_L64(rsp, 16)); + WC_S64(r10, 48) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 24)); + WC_S64(r10, 56) = (word64)(rax); + rax = (word64)(0); + WC_S64(r10, 352) = (word64)(rax); + WC_S64(r10, 408) = (word64)(rax); + WC_S64(r10, 360) = (word64)(r11); + WC_S64(r10, 416) = (word64)(r12); + rcx = (word64)(r11); + rsi = (word64)(r12); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 368) = (word64)(rcx); + WC_S64(r10, 424) = (word64)(rsi); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 376) = (word64)(rcx); + WC_S64(r10, 432) = (word64)(rsi); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 384) = (word64)(rcx); + WC_S64(r10, 440) = (word64)(rsi); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 392) = (word64)(rcx); + WC_S64(r10, 448) = (word64)(rsi); + rcx = (word64)(rcx + r11); + rsi = (word64)(rsi + r12); + WC_S64(r10, 400) = (word64)(rcx); + WC_S64(r10, 456) = (word64)(rsi); + r13 = (word64)(0); + r14 = (word64)(0); + r15 = (word64)(0); + rbx = (word64)(r9); + rbp = (word64)(WC_L64(rsp, 128)); +L_c20p1305_small_dec_aad_full: + if ((sword64)(rbp) < (sword64)(0x10)) { + goto L_c20p1305_small_dec_aad_part; + } + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rbx = (word64)(rbx + 0x10); + rbp = (word64)(rbp - 0x10); + goto L_c20p1305_small_dec_aad_full; +L_c20p1305_small_dec_aad_part: + if (((rbp & rbp)) == (0)) { + goto L_c20p1305_small_dec_aad_done; + } + rax = (word64)(0); + WC_S64(rsp, 96) = (word64)(rax); + WC_S64(rsp, 104) = (word64)(rax); + r9 = (word64)(rsp + 96); + if ((sword64)(rbp) < (sword64)(8)) { + goto L_c20p1305_small_dec_aad_c4; + } + rax = (word64)(WC_L64(rbx, 0)); + WC_S64(r9, 0) = (word64)(rax); + rbx = (word64)(rbx + 8); + r9 = (word64)(r9 + 8); + rbp = (word64)(rbp - 8); +L_c20p1305_small_dec_aad_c4: + if ((sword64)(rbp) < (sword64)(4)) { + goto L_c20p1305_small_dec_aad_cby; + } + rax = (word32)(WC_L32(rbx, 0)); + WC_S32(r9, 0) = (word32)((word32)rax); + rbx = (word64)(rbx + 4); + r9 = (word64)(r9 + 4); + rbp = (word64)(rbp - 4); +L_c20p1305_small_dec_aad_cby: + if (((rbp & rbp)) == (0)) { + goto L_c20p1305_small_dec_aad_cpyd; + } +L_c20p1305_small_dec_aad_cbyl: + rax = (word32)((word32)WC_L8(rbx, 0)); + WC_S8(r9, 0) = (byte)((byte)rax); + rbx = (word64)(rbx + 1); + r9 = (word64)(r9 + 1); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_c20p1305_small_dec_aad_cbyl; + } +L_c20p1305_small_dec_aad_cpyd: + rbx = (word64)(rsp + 96); + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); +L_c20p1305_small_dec_aad_done: + rbx = (word64)(WC_L64(rsp, 152)); + rbp = (word64)(WC_L64(rsp, 136)); +L_c20p1305_small_dec_ct_full: + if ((sword64)(rbp) < (sword64)(0x10)) { + goto L_c20p1305_small_dec_ct_part; + } + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rbx = (word64)(rbx + 0x10); + rbp = (word64)(rbp - 0x10); + goto L_c20p1305_small_dec_ct_full; +L_c20p1305_small_dec_ct_part: + if (((rbp & rbp)) == (0)) { + goto L_c20p1305_small_dec_ct_done; + } + rax = (word64)(0); + WC_S64(rsp, 96) = (word64)(rax); + WC_S64(rsp, 104) = (word64)(rax); + r9 = (word64)(rsp + 96); + if ((sword64)(rbp) < (sword64)(8)) { + goto L_c20p1305_small_dec_ct_c4; + } + rax = (word64)(WC_L64(rbx, 0)); + WC_S64(r9, 0) = (word64)(rax); + rbx = (word64)(rbx + 8); + r9 = (word64)(r9 + 8); + rbp = (word64)(rbp - 8); +L_c20p1305_small_dec_ct_c4: + if ((sword64)(rbp) < (sword64)(4)) { + goto L_c20p1305_small_dec_ct_cby; + } + rax = (word32)(WC_L32(rbx, 0)); + WC_S32(r9, 0) = (word32)((word32)rax); + rbx = (word64)(rbx + 4); + r9 = (word64)(r9 + 4); + rbp = (word64)(rbp - 4); +L_c20p1305_small_dec_ct_cby: + if (((rbp & rbp)) == (0)) { + goto L_c20p1305_small_dec_ct_cpyd; + } +L_c20p1305_small_dec_ct_cbyl: + rax = (word32)((word32)WC_L8(rbx, 0)); + WC_S8(r9, 0) = (byte)((byte)rax); + rbx = (word64)(rbx + 1); + r9 = (word64)(r9 + 1); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_c20p1305_small_dec_ct_cbyl; + } +L_c20p1305_small_dec_ct_cpyd: + rbx = (word64)(rsp + 96); + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); +L_c20p1305_small_dec_ct_done: + rax = (word64)(WC_L64(rsp, 128)); + WC_S64(rsp, 112) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 136)); + WC_S64(rsp, 120) = (word64)(rax); + rbx = (word64)(rsp + 112); + rcx = (word64)(WC_L64(rbx, 0)); + rsi = (word64)(WC_L64(rbx, 8)); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + rax = (word64)(r12); + cf = _addcarry_u64(cf, r15, 1, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, r13); + rsi = (word64)(rax); + rdi = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + rax = (word64)(r11); + cf = _addcarry_u64(cf, rdi, rdx, (unsigned long long*)&rdi); + WC_X64I_MUL128(rax, rdx, rax, r13); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r14); + rdi = (word64)(rdi + WC_L64(r10, r15 * 8 + 352)); + r8 = (word64)(rdx); + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, rax, (unsigned long long*)&rdi); + cf = _addcarry_u64(cf, r8, WC_L64(r10, r15 * 8 + 408), ( + unsigned long long*)&r8); + r15 = (word64)(rdi); + rdi = (word64)(rdi & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, rcx, rdi, (unsigned long long*)&rcx); + r13 = (word64)(rdi); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + r13 = (word64)((r13 >> 2) | (r8 << 62)); + r8 = (word64)(r8 >> 2); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + r14 = (word64)(rsi); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rdx = (word64)(WC_L64(rsp, 152)); + rcx = (word64)(WC_L64(rsp, 160)); + rbp = (word64)(WC_L64(rsp, 136)); + rbx = (word64)(rsp + 32); +L_chacha20_poly1305_small_dec_x16: + if ((sword64)(rbp) < (sword64)(0x10)) { + goto L_chacha20_poly1305_small_dec_xtail; + } + y8 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + y8 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y8), + _mm_loadu_si128((const __m128i*)WC_PR(rbx, 0)))); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y8)); + rdx = (word64)(rdx + 0x10); + rcx = (word64)(rcx + 0x10); + rbx = (word64)(rbx + 0x10); + rbp = (word64)(rbp - 0x10); + goto L_chacha20_poly1305_small_dec_x16; +L_chacha20_poly1305_small_dec_xtail: + if (((rbp & rbp)) == (0)) { + goto L_chacha20_poly1305_small_dec_xdone; + } + rdi = (word64)(0); +L_chacha20_poly1305_small_dec_xbyte: + if ((sword64)(rdi) >= (sword64)(rbp)) { + goto L_chacha20_poly1305_small_dec_xdone; + } + rax = (word32)((word32)WC_L8(rdx, rdi)); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax ^ WC_L8(rbx, + rdi)) & 0xff); + WC_S8(rcx, rdi) = (byte)((byte)rax); + rdi = (word64)(rdi + 1); + goto L_chacha20_poly1305_small_dec_xbyte; +L_chacha20_poly1305_small_dec_xdone: + y0 = _mm256_zextsi128_si256(_mm_setzero_si128()); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), _mm256_castsi256_si128(y0)); + rbx = (word64)(WC_L64(rsp, 144)); + rcx = (word64)(r15); + r15 = (word64)(r15 & 3); + rcx = (word64)(rcx >> 2); + rcx = (word64)(rcx + rcx * 4); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rcx = (word64)(r13); + rsi = (word64)(r14); + rdi = (word64)(r15); + cf = _addcarry_u64(0, rcx, 5, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rdi, 0, (unsigned long long*)&rdi); + zf1 = rdi; + zf2 = 4; + r13 = (rdi) == (4) ? rcx : r13; + r14 = (zf1) == (zf2) ? rsi : r14; + cf = _addcarry_u64(0, r13, WC_L64(r10, 48), (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, WC_L64(r10, 56), (unsigned long long*)&r14); + r13 = (word64)(r13 ^ WC_L64(rbx, 0)); + r14 = (word64)(r14 ^ WC_L64(rbx, 8)); + r13 = (word64)(r13 | r14); + rax = (word64)(r13); + rax = (word64)(0 - rax); + rax = (word64)(rax | r13); + rax = (word64)(rax >> 63); + return (int)(word32)rax; + (void)aadLen; + (void)tag; +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* WOLFSSL_X86_64_BUILD */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/fe_x25519_asm.asm b/wolfcrypt/src/fe_x25519_asm.asm index cbbb99a44e5..3bd50cd61c3 100644 --- a/wolfcrypt/src/fe_x25519_asm.asm +++ b/wolfcrypt/src/fe_x25519_asm.asm @@ -1290,116 +1290,178 @@ fe_invert_x64 PROC mov QWORD PTR [rsp+136], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+136] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+136] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rdx, QWORD PTR [rsp+136] mov rcx, QWORD PTR [rsp+128] add rsp, 152 @@ -3314,116 +3376,178 @@ L_curve25519_base_x64_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov r8, QWORD PTR [rsp+160] ; Multiply ; A[0] * B[0] @@ -5609,116 +5733,178 @@ L_curve25519_x64_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov r9, QWORD PTR [rsp+168] ; Multiply ; A[0] * B[0] @@ -5899,115 +6085,177 @@ fe_pow22523_x64 PROC mov QWORD PTR [rsp+104], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+104] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+104] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, QWORD PTR [rsp+96] mov rdx, rsp mov r8, QWORD PTR [rsp+104] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rdx, QWORD PTR [rsp+104] mov rcx, QWORD PTR [rsp+96] add rsp, 120 @@ -11543,116 +11791,178 @@ fe_invert_avx2 PROC mov QWORD PTR [rsp+136], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+136] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+136] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rdx, QWORD PTR [rsp+136] mov rcx, QWORD PTR [rsp+128] add rsp, 152 @@ -13206,116 +13516,178 @@ L_curve25519_base_avx2_last_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov r8, QWORD PTR [rsp+160] mov rax, QWORD PTR [r8] ; Multiply @@ -15095,116 +15467,178 @@ L_curve25519_avx2_last_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov r9, QWORD PTR [rsp+168] mov rax, QWORD PTR [r9] ; Multiply @@ -15356,115 +15790,177 @@ fe_pow22523_avx2 PROC mov QWORD PTR [rsp+104], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+104] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+104] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, QWORD PTR [rsp+96] mov rdx, rsp mov r8, QWORD PTR [rsp+104] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rdx, QWORD PTR [rsp+104] mov rcx, QWORD PTR [rsp+96] add rsp, 120 @@ -20577,16 +21073,22 @@ L_curve25519_base_avx512_ifma_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+736] vmovdqu xmm7, OWORD PTR [rsp+752] @@ -21395,16 +21897,22 @@ L_curve25519_avx512_ifma_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+744] vmovdqu xmm7, OWORD PTR [rsp+760] @@ -22147,16 +22655,22 @@ L_curve25519_base_avx512_ifma_dq_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+736] vmovdqu xmm7, OWORD PTR [rsp+752] @@ -22920,16 +23434,22 @@ L_curve25519_avx512_ifma_dq_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+744] vmovdqu xmm7, OWORD PTR [rsp+760] diff --git a/wolfcrypt/src/fe_x25519_intrin.c b/wolfcrypt/src/fe_x25519_intrin.c new file mode 100644 index 00000000000..f348365c6af --- /dev/null +++ b/wolfcrypt/src/fe_x25519_intrin.c @@ -0,0 +1,21710 @@ +/* fe_x25519_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_FE_X25519_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +extern WOLFSSL_LOCAL void fe_init(void); +extern WOLFSSL_LOCAL void fe_frombytes(fe out, const unsigned char* in); +extern WOLFSSL_LOCAL void fe_tobytes(unsigned char* out, const fe n); +extern WOLFSSL_LOCAL void fe_1(fe n); +extern WOLFSSL_LOCAL void fe_0(fe n); +extern WOLFSSL_LOCAL void fe_copy(fe r, const fe a); +extern WOLFSSL_LOCAL void fe_sub(fe r, const fe a, const fe b); +extern WOLFSSL_LOCAL void fe_add(fe r, const fe a, const fe b); +extern WOLFSSL_LOCAL void fe_neg(fe r, const fe a); +extern WOLFSSL_LOCAL void fe_cmov(fe a, const fe b, int c); +extern WOLFSSL_LOCAL int fe_isnonzero(const fe a); +extern WOLFSSL_LOCAL int fe_isnegative(const fe a); +extern WOLFSSL_LOCAL void fe_cmov_table(fe* r, const fe* base, signed char b); +extern WOLFSSL_LOCAL void fe_mul(fe r, const fe a, const fe b); +extern WOLFSSL_LOCAL void fe_sq(fe r, const fe a); +extern WOLFSSL_LOCAL void fe_mul121666(fe r, fe a); +extern WOLFSSL_LOCAL void fe_invert(fe r, const fe a); +extern WOLFSSL_LOCAL int curve25519(byte* r, const byte* n, const byte* a); +extern WOLFSSL_LOCAL void fe_pow22523(fe r, const fe a); +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +extern WOLFSSL_LOCAL void ge_p1p1_to_p2(ge_p2* r, const ge_p1p1* p); +extern WOLFSSL_LOCAL void ge_p1p1_to_p3(ge_p3* r, const ge_p1p1* p); +extern WOLFSSL_LOCAL void ge_p2_dbl(ge_p1p1* r, const ge_p2* p); +extern WOLFSSL_LOCAL void ge_madd(ge_p1p1* r, const ge_p3* p, + const ge_precomp* q); +extern WOLFSSL_LOCAL void ge_msub(ge_p1p1* r, const ge_p3* p, + const ge_precomp* q); +extern WOLFSSL_LOCAL void ge_add(ge_p1p1* r, const ge_p3* p, + const ge_cached* q); +extern WOLFSSL_LOCAL void ge_sub(ge_p1p1* r, const ge_p3* p, + const ge_cached* q); +#endif +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +extern WOLFSSL_LOCAL int curve25519_base(byte* r, const byte* n); +#endif +#endif +#ifdef HAVE_ED25519 +#ifdef HAVE_ED25519 +extern WOLFSSL_LOCAL void fe_sq2(fe r, const fe a); +extern WOLFSSL_LOCAL void fe_invert_nct(fe r, const fe a); +extern WOLFSSL_LOCAL void sc_reduce(byte* s); +extern WOLFSSL_LOCAL void sc_muladd(byte* s, const byte* a, const byte* b, + const byte* c); +#endif +#endif +extern WOLFSSL_LOCAL void fe_cmov_table_x64(fe* r, const fe* base, + signed char b); +extern WOLFSSL_LOCAL void fe_mul_x64(fe r, fe a, fe b); +extern WOLFSSL_LOCAL void fe_sq_x64(fe r, fe a); +extern WOLFSSL_LOCAL void fe_sq_n_x64(fe r, const fe a, word64 n); +extern WOLFSSL_LOCAL void fe_mul121666_x64(fe r, fe a); +extern WOLFSSL_LOCAL void fe_invert_x64(fe r, const fe a); +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +extern WOLFSSL_LOCAL int curve25519_base_x64(byte* r, byte* n); +#endif +extern WOLFSSL_LOCAL int curve25519_x64(byte* r, byte* n, byte* a); +extern WOLFSSL_LOCAL void fe_pow22523_x64(fe r, const fe a); +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +extern WOLFSSL_LOCAL void ge_p1p1_to_p2_x64(ge_p2* r, const ge_p1p1* p); +extern WOLFSSL_LOCAL void ge_p1p1_to_p3_x64(ge_p3* r, const ge_p1p1* p); +extern WOLFSSL_LOCAL void ge_p2_dbl_x64(ge_p1p1* r, const ge_p2* p); +extern WOLFSSL_LOCAL void ge_madd_x64(ge_p1p1* r, const ge_p3* p, + const ge_precomp* q); +extern WOLFSSL_LOCAL void ge_msub_x64(ge_p1p1* r, const ge_p3* p, + const ge_precomp* q); +extern WOLFSSL_LOCAL void ge_add_x64(ge_p1p1* r, const ge_p3* p, + const fe qe_cached); +extern WOLFSSL_LOCAL void ge_sub_x64(ge_p1p1* r, const ge_p3* p, + const fe qe_cached); +#endif +#ifdef HAVE_ED25519 +extern WOLFSSL_LOCAL void fe_sq2_x64(fe r, fe a); +extern WOLFSSL_LOCAL void sc_reduce_x64(byte* s); +extern WOLFSSL_LOCAL void sc_muladd_x64(byte* s, byte* a, byte* b, byte* c); +extern WOLFSSL_LOCAL void fe_invert_nct_x64(word64* r, const word64* a); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void fe_cmov_table_avx2(fe* r, const fe* base, + signed char b); +extern WOLFSSL_LOCAL void fe_mul_avx2(fe r, fe a, fe b); +extern WOLFSSL_LOCAL void fe_sq_avx2(fe r, fe a); +extern WOLFSSL_LOCAL void fe_sq_n_avx2(fe r, const fe a, word64 n); +extern WOLFSSL_LOCAL void fe_mul121666_avx2(fe r, fe a); +extern WOLFSSL_LOCAL void fe_invert_avx2(fe r, const fe a); +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +extern WOLFSSL_LOCAL int curve25519_base_avx2(byte* r, byte* n); +#endif +extern WOLFSSL_LOCAL int curve25519_avx2(byte* r, byte* n, byte* a); +extern WOLFSSL_LOCAL void fe_pow22523_avx2(fe r, const fe a); +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +extern WOLFSSL_LOCAL void ge_p1p1_to_p2_avx2(ge_p2* r, const ge_p1p1* p); +extern WOLFSSL_LOCAL void ge_p1p1_to_p3_avx2(ge_p3* r, const ge_p1p1* p); +extern WOLFSSL_LOCAL void ge_p2_dbl_avx2(ge_p1p1* r, const ge_p2* p); +extern WOLFSSL_LOCAL void ge_madd_avx2(ge_p1p1* r, const ge_p3* p, + const ge_precomp* q); +extern WOLFSSL_LOCAL void ge_msub_avx2(ge_p1p1* r, const ge_p3* p, + const ge_precomp* q); +extern WOLFSSL_LOCAL void ge_add_avx2(ge_p1p1* r, const ge_p3* p, + const fe qe_cached); +extern WOLFSSL_LOCAL void ge_sub_avx2(ge_p1p1* r, const ge_p3* p, + const fe qe_cached); +#endif +#ifdef HAVE_ED25519 +extern WOLFSSL_LOCAL void fe_sq2_avx2(fe r, fe a); +extern WOLFSSL_LOCAL void sc_reduce_avx2(byte* s); +extern WOLFSSL_LOCAL void sc_muladd_avx2(byte* s, byte* a, byte* b, byte* c); +extern WOLFSSL_LOCAL void fe_invert_nct_avx2(word64* r, const word64* a); +#endif +#ifdef HAVE_INTEL_AVX512_IFMA +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +extern WOLFSSL_LOCAL int curve25519_base_avx512_ifma(byte* r, byte* n); +#endif +extern WOLFSSL_LOCAL int curve25519_avx512_ifma(byte* r, byte* n, byte* a); +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +extern WOLFSSL_LOCAL int curve25519_base_avx512_ifma_dq(byte* r, byte* n); +#endif +extern WOLFSSL_LOCAL int curve25519_avx512_ifma_dq(byte* r, byte* n, byte* a); +#ifdef HAVE_ED25519 +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +extern WOLFSSL_LOCAL int ge_double_scalarmult_vartime_avx512_ifma(ge_p2* r, + const byte* a, const ge_p3* A, const byte* b, const ge_precomp* bi, + byte* buf); +extern WOLFSSL_LOCAL int ge_double_scalarmult_vartime_avx512_ifma_dq(ge_p2* r, + const byte* a, const ge_p3* A, const byte* b, const ge_precomp* bi, + byte* buf); + +#endif +#endif +#endif +#endif +static word32 cpuFlagsSet = 0; +static word32 intelFlags = 0; +static word64 fe_cmov_table_p = (word64)(size_t)fe_cmov_table_x64; +static word64 fe_mul_p = (word64)(size_t)fe_mul_x64; +static word64 fe_sq_p = (word64)(size_t)fe_sq_x64; +static word64 fe_mul121666_p = (word64)(size_t)fe_mul121666_x64; +static word64 fe_invert_p = (word64)(size_t)fe_invert_x64; +static word64 curve25519_p = (word64)(size_t)curve25519_x64; +static word64 fe_pow22523_p = (word64)(size_t)fe_pow22523_x64; +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +static word64 ge_p1p1_to_p2_p = (word64)(size_t)ge_p1p1_to_p2_x64; +static word64 ge_p1p1_to_p3_p = (word64)(size_t)ge_p1p1_to_p3_x64; +static word64 ge_p2_dbl_p = (word64)(size_t)ge_p2_dbl_x64; +static word64 ge_madd_p = (word64)(size_t)ge_madd_x64; +static word64 ge_msub_p = (word64)(size_t)ge_msub_x64; +static word64 ge_add_p = (word64)(size_t)ge_add_x64; +static word64 ge_sub_p = (word64)(size_t)ge_sub_x64; +#endif +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +static word64 curve25519_base_p = (word64)(size_t)curve25519_base_x64; +#endif +#ifdef HAVE_ED25519 +static word64 fe_sq2_p = (word64)(size_t)fe_sq2_x64; +static word64 fe_invert_nct_p = (word64)(size_t)fe_invert_nct_x64; +static word64 sc_reduce_p = (word64)(size_t)sc_reduce_x64; +static word64 sc_muladd_p = (word64)(size_t)sc_muladd_x64; + +#endif +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ +WOLFSSL_LOCAL void fe_init(void) +{ + word64 rax; + +#ifdef HAVE_INTEL_AVX2 + rax = (word32)((word32)cpuFlagsSet); + if ((((word32)rax & (word32)rax)) == (0)) { + goto L_fe_init_get_flags; + } + return; +L_fe_init_get_flags: + rax = (word64)cpuid_get_flags(); + intelFlags = (word32)(word32)rax; + rax = (word32)((word32)rax & 0x50); + if (((word32)rax) != (0x50)) { + goto L_fe_init_flags_done; + } + rax = (word64)((word64)(size_t)fe_cmov_table_avx2); + fe_cmov_table_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)fe_mul_avx2); + fe_mul_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)fe_sq_avx2); + fe_sq_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)fe_mul121666_avx2); + fe_mul121666_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)fe_invert_avx2); + fe_invert_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)curve25519_avx2); + curve25519_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)fe_pow22523_avx2); + fe_pow22523_p = (word64)(size_t)rax; +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + rax = (word64)((word64)(size_t)ge_p1p1_to_p2_avx2); + ge_p1p1_to_p2_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)ge_p1p1_to_p3_avx2); + ge_p1p1_to_p3_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)ge_p2_dbl_avx2); + ge_p2_dbl_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)ge_madd_avx2); + ge_madd_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)ge_msub_avx2); + ge_msub_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)ge_add_avx2); + ge_add_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)ge_sub_avx2); + ge_sub_p = (word64)(size_t)rax; +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) + rax = (word64)((word64)(size_t)curve25519_base_avx2); + curve25519_base_p = (word64)(size_t)rax; +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ +#ifdef HAVE_ED25519 + rax = (word64)((word64)(size_t)fe_sq2_avx2); + fe_sq2_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)fe_invert_nct_avx2); + fe_invert_nct_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)sc_reduce_avx2); + sc_reduce_p = (word64)(size_t)rax; + rax = (word64)((word64)(size_t)sc_muladd_avx2); + sc_muladd_p = (word64)(size_t)rax; +#endif /* HAVE_ED25519 */ +#ifdef HAVE_INTEL_AVX512_IFMA + rax = (word32)((word32)intelFlags); + rax = (word32)((word32)rax & 0x30800); + if (((word32)rax) != (0x30800)) { + goto L_fe_init_flags_done; + } + rax = (word64)((word64)(size_t)curve25519_avx512_ifma); + curve25519_p = (word64)(size_t)rax; +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) + rax = (word64)((word64)(size_t)curve25519_base_avx512_ifma); + curve25519_base_p = (word64)(size_t)rax; +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ + rax = (word32)((word32)intelFlags); + rax = (word32)((word32)rax & 0x72800); + if (((word32)rax) != (0x72800)) { + goto L_fe_init_flags_done; + } + rax = (word64)((word64)(size_t)curve25519_avx512_ifma_dq); + curve25519_p = (word64)(size_t)rax; +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) + rax = (word64)((word64)(size_t)curve25519_base_avx512_ifma_dq); + curve25519_base_p = (word64)(size_t)rax; +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ +#endif /* HAVE_INTEL_AVX512_IFMA */ +L_fe_init_flags_done: + cpuFlagsSet = (word32)1; +#endif /* HAVE_INTEL_AVX2 */ +} + +WOLFSSL_LOCAL void fe_frombytes(fe out, const unsigned char* in) +{ + word64 rdi, rsi, r9, rdx, rax, rcx, r8; + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)in; + + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r8 = (word64)(r8 & r9); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); +} + +WOLFSSL_LOCAL void fe_tobytes(unsigned char* out, const fe n) +{ + word64 rdi, rsi, r10, rdx, rax, rcx, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)n; + + r10 = (word64)(0x7fffffffffffffff); + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rdx, 0x13, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + r8 = (word64)(r8 >> 63); + r9 = (word64)(r8 * 19); + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + r8 = (word64)(r8 & r10); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); +} + +WOLFSSL_LOCAL void fe_1(fe n) +{ + word64 rdi; + + rdi = (word64)(size_t)n; + + /* Set one */ + WC_S64(rdi, 0) = (word64)(1); + WC_S64(rdi, 8) = (word64)(0); + WC_S64(rdi, 16) = (word64)(0); + WC_S64(rdi, 24) = (word64)(0); +} + +WOLFSSL_LOCAL void fe_0(fe n) +{ + word64 rdi; + + rdi = (word64)(size_t)n; + + /* Set zero */ + WC_S64(rdi, 0) = (word64)(0); + WC_S64(rdi, 8) = (word64)(0); + WC_S64(rdi, 16) = (word64)(0); + WC_S64(rdi, 24) = (word64)(0); +} + +WOLFSSL_LOCAL void fe_copy(fe r, const fe a) +{ + word64 rdi, rsi, rdx, rax, rcx, r8; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + /* Copy */ + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); +} + +WOLFSSL_LOCAL void fe_sub(fe r, const fe a, const fe b) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10 = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Sub */ + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + cf = _subborrow_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 8), (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 16), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 24), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r10, (unsigned long long*)&r10); + r10 = (word64)((r10 << 1) | (r9 >> 63)); + r10 = (word64)(r10 * -19); + r9 = (word64)(r9 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, rax, r10, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); +} + +WOLFSSL_LOCAL void fe_add(fe r, const fe a, const fe b) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Add */ + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + cf = _addcarry_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + r8 = (word64)(WC_L64(rsi, 16)); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 8), (unsigned long long*)&rcx); + r9 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 16), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 24), (unsigned long long*)&r9); + r10 = (word64)(0); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r10 = (word64)((r10 << 1) | (r9 >> 63)); + r10 = (word64)(r10 * 0x13); + r9 = (word64)(r9 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, rax, r10, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); +} + +WOLFSSL_LOCAL void fe_neg(fe r, const fe a) +{ + word64 rdi, rsi, rdx, rax, rcx, r8; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(-19); + rax = (word64)(-1); + rcx = (word64)(-1); + r8 = (word64)(0x7fffffffffffffff); + cf = _subborrow_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 8), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 16), (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rsi, 24), (unsigned long long*)&r8); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); +} + +WOLFSSL_LOCAL void fe_cmov(fe a, const fe b, int c) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, r10; + word32 zf1; + word32 zf2; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + rdx = (word64)(word64)c; + + zf1 = (word32)rdx; + zf2 = 1; + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rdi, 24)); + rcx = (zf1) == (zf2) ? WC_L64(rsi, 0) : rcx; + r8 = (zf1) == (zf2) ? WC_L64(rsi, 8) : r8; + r9 = (zf1) == (zf2) ? WC_L64(rsi, 16) : r9; + r10 = (zf1) == (zf2) ? WC_L64(rsi, 24) : r10; + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); +} + +WOLFSSL_LOCAL int fe_isnonzero(const fe a) +{ + word64 rdi, r10, rax, rdx, rcx, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)a; + + r10 = (word64)(0x7fffffffffffffff); + rax = (word64)(WC_L64(rdi, 0)); + rdx = (word64)(WC_L64(rdi, 8)); + rcx = (word64)(WC_L64(rdi, 16)); + r8 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rax, 0x13, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + r8 = (word64)(r8 >> 63); + r9 = (word64)(r8 * 19); + rax = (word64)(WC_L64(rdi, 0)); + rdx = (word64)(WC_L64(rdi, 8)); + rcx = (word64)(WC_L64(rdi, 16)); + r8 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + r8 = (word64)(r8 & r10); + rax = (word64)(rax | rdx); + rax = (word64)(rax | rcx); + rax = (word64)(rax | r8); + return (int)(word32)rax; +} + +WOLFSSL_LOCAL int fe_isnegative(const fe a) +{ + word64 rdi, r11, rdx, rcx, r8, r9, rax, r10; + unsigned char cf; + + rdi = (word64)(size_t)a; + + r11 = (word64)(0x7fffffffffffffff); + rdx = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)(WC_L64(rdi, 16)); + r9 = (word64)(WC_L64(rdi, 24)); + rax = (word64)(rdx); + cf = _addcarry_u64(0, rdx, 0x13, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)(r9 >> 63); + r10 = (word64)(r9 * 19); + rax = (word64)(rax + r10); + rax = (word64)(rax & 1); + return (int)(word32)rax; + (void)r11; +} + +WOLFSSL_LOCAL void fe_cmov_table(fe* r, const fe* base, signed char b) +{ + ((void(*)(fe*, const fe*, signed char))(size_t)fe_cmov_table_p)(r, base, b); + return; + (void)r; + (void)base; + (void)b; +} + +WOLFSSL_LOCAL void fe_mul(fe r, const fe a, const fe b) +{ + ((void(*)(fe, const fe, const fe))(size_t)fe_mul_p)(r, a, b); + return; + (void)r; + (void)a; + (void)b; +} + +WOLFSSL_LOCAL void fe_sq(fe r, const fe a) +{ + ((void(*)(fe, const fe))(size_t)fe_sq_p)(r, a); + return; + (void)r; + (void)a; +} + +WOLFSSL_LOCAL void fe_mul121666(fe r, fe a) +{ + ((void(*)(fe, fe))(size_t)fe_mul121666_p)(r, a); + return; + (void)r; + (void)a; +} + +WOLFSSL_LOCAL void fe_invert(fe r, const fe a) +{ + ((void(*)(fe, const fe))(size_t)fe_invert_p)(r, a); + return; + (void)r; + (void)a; +} + +WOLFSSL_LOCAL int curve25519(byte* r, const byte* n, const byte* a) +{ + return ((int(*)(byte*, const byte*, const byte*))(size_t)curve25519_p)(r, n, + a); + (void)r; + (void)n; + (void)a; +} + +WOLFSSL_LOCAL void fe_pow22523(fe r, const fe a) +{ + ((void(*)(fe, const fe))(size_t)fe_pow22523_p)(r, a); + return; + (void)r; + (void)a; +} + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_p1p1_to_p2(ge_p2* r, const ge_p1p1* p) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ((void(*)(ge_p2*, const ge_p1p1*))(size_t)ge_p1p1_to_p2_p)(r, p); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + return; + (void)r; + (void)p; +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_p1p1_to_p3(ge_p3* r, const ge_p1p1* p) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ((void(*)(ge_p3*, const ge_p1p1*))(size_t)ge_p1p1_to_p3_p)(r, p); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + return; + (void)r; + (void)p; +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_p2_dbl(ge_p1p1* r, const ge_p2* p) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ((void(*)(ge_p1p1*, const ge_p2*))(size_t)ge_p2_dbl_p)(r, p); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + return; + (void)r; + (void)p; +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_madd(ge_p1p1* r, const ge_p3* p, const ge_precomp* q) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ((void(*)(ge_p1p1*, const ge_p3*, const ge_precomp*))(size_t)ge_madd_p)(r, +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + p, q); + return; + (void)r; + (void)p; + (void)q; +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_msub(ge_p1p1* r, const ge_p3* p, const ge_precomp* q) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ((void(*)(ge_p1p1*, const ge_p3*, const ge_precomp*))(size_t)ge_msub_p)(r, +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + p, q); + return; + (void)r; + (void)p; + (void)q; +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_add(ge_p1p1* r, const ge_p3* p, const ge_cached* q) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ((void(*)(ge_p1p1*, const ge_p3*, const ge_cached*))(size_t)ge_add_p)(r, p, +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + q); + return; + (void)r; + (void)p; + (void)q; +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_sub(ge_p1p1* r, const ge_p3* p, const ge_cached* q) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ((void(*)(ge_p1p1*, const ge_p3*, const ge_cached*))(size_t)ge_sub_p)(r, p, +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + q); + return; + (void)r; + (void)p; + (void)q; +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +WOLFSSL_LOCAL int curve25519_base(byte* r, const byte* n) +{ + return ((int(*)(byte*, const byte*))(size_t)curve25519_base_p)(r, n); + (void)r; + (void)n; +} + +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ +#ifdef HAVE_ED25519 +#ifdef HAVE_ED25519 +WOLFSSL_LOCAL void fe_sq2(fe r, const fe a) +{ + ((void(*)(fe, const fe))(size_t)fe_sq2_p)(r, a); + return; + (void)r; + (void)a; +} + +#endif /* HAVE_ED25519 */ +#ifdef HAVE_ED25519 +WOLFSSL_LOCAL void fe_invert_nct(fe r, const fe a) +{ + ((void(*)(fe, const fe))(size_t)fe_invert_nct_p)(r, a); + return; + (void)r; + (void)a; +} + +#endif /* HAVE_ED25519 */ +#ifdef HAVE_ED25519 +WOLFSSL_LOCAL void sc_reduce(byte* s) +{ + ((void(*)(byte*))(size_t)sc_reduce_p)(s); + return; + (void)s; +} + +#endif /* HAVE_ED25519 */ +#ifdef HAVE_ED25519 +WOLFSSL_LOCAL void sc_muladd(byte* s, const byte* a, const byte* b, + const byte* c) +{ + ((void(*)(byte*, const byte*, const byte*, const byte*))( + size_t)sc_muladd_p)(s, a, b, c); + return; + (void)s; + (void)a; + (void)b; + (void)c; +} + +#endif /* HAVE_ED25519 */ +#endif /* HAVE_ED25519 */ +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ +#ifdef HAVE_ED25519 +#endif /* HAVE_ED25519 */ +WOLFSSL_LOCAL void fe_cmov_table_x64(fe* r, const fe* base, signed char b) +{ + word64 rdi, rsi, rcx, rax, rdx, r15 = 0, r8, r9, r10, r11, r12, r13, r14; + byte zf1; + byte zf2; + byte zf3; + byte zf4; + byte zf5; + byte zf6; + byte zf7; + byte zf8; + byte zf9; + byte zf10; + byte zf11; + byte zf12; + byte zf13; + byte zf14; + byte zf15; + byte zf16; + byte zf17; + byte zf18; + byte zf19; + byte zf20; + byte zf21; + byte zf22; + byte zf23; + byte zf24; + byte zf25; + byte zf26; + byte zf27; + byte zf28; + byte zf29; + byte zf30; + byte zf31; + byte zf32; + byte zf33; + byte zf34; + byte zf35; + byte zf36; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)base; + rcx = (word64)(byte)b; + + rax = (word64)((word64)(sword64)(signed char)(byte)rcx); + rdx = (word64)(word32)((sword32)(word32)rax >> 31); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax ^ ( + byte)rdx) & 0xff); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax - ( + byte)rdx) & 0xff); + r15 = (r15 & ~(word64)0xff) | ((word64)(byte)((byte)rax) & 0xff); + rax = (word64)(1); + rdx = (word64)(0); + r8 = (word64)(0); + r9 = (word64)(0); + r10 = (word64)(1); + r11 = (word64)(0); + r12 = (word64)(0); + r13 = (word64)(0); + zf1 = (byte)r15; + zf2 = 1; + r14 = (word64)(WC_L64(rsi, 0)); + rax = (zf1) == (zf2) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 8)); + rdx = (zf1) == (zf2) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 16)); + r8 = (zf1) == (zf2) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 24)); + r9 = (zf1) == (zf2) ? r14 : r9; + r14 = (word64)(WC_L64(rsi, 32)); + r10 = (zf1) == (zf2) ? r14 : r10; + r14 = (word64)(WC_L64(rsi, 40)); + r11 = (zf1) == (zf2) ? r14 : r11; + r14 = (word64)(WC_L64(rsi, 48)); + r12 = (zf1) == (zf2) ? r14 : r12; + r14 = (word64)(WC_L64(rsi, 56)); + r13 = (zf1) == (zf2) ? r14 : r13; + zf3 = (byte)r15; + zf4 = 2; + r14 = (word64)(WC_L64(rsi, 96)); + rax = (zf3) == (zf4) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 104)); + rdx = (zf3) == (zf4) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 112)); + r8 = (zf3) == (zf4) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 120)); + r9 = (zf3) == (zf4) ? r14 : r9; + r14 = (word64)(WC_L64(rsi, 128)); + r10 = (zf3) == (zf4) ? r14 : r10; + r14 = (word64)(WC_L64(rsi, 136)); + r11 = (zf3) == (zf4) ? r14 : r11; + r14 = (word64)(WC_L64(rsi, 144)); + r12 = (zf3) == (zf4) ? r14 : r12; + r14 = (word64)(WC_L64(rsi, 152)); + r13 = (zf3) == (zf4) ? r14 : r13; + zf5 = (byte)r15; + zf6 = 3; + r14 = (word64)(WC_L64(rsi, 192)); + rax = (zf5) == (zf6) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 200)); + rdx = (zf5) == (zf6) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 208)); + r8 = (zf5) == (zf6) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 216)); + r9 = (zf5) == (zf6) ? r14 : r9; + r14 = (word64)(WC_L64(rsi, 224)); + r10 = (zf5) == (zf6) ? r14 : r10; + r14 = (word64)(WC_L64(rsi, 232)); + r11 = (zf5) == (zf6) ? r14 : r11; + r14 = (word64)(WC_L64(rsi, 240)); + r12 = (zf5) == (zf6) ? r14 : r12; + r14 = (word64)(WC_L64(rsi, 248)); + r13 = (zf5) == (zf6) ? r14 : r13; + zf7 = (byte)r15; + zf8 = 4; + r14 = (word64)(WC_L64(rsi, 288)); + rax = (zf7) == (zf8) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 296)); + rdx = (zf7) == (zf8) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 304)); + r8 = (zf7) == (zf8) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 312)); + r9 = (zf7) == (zf8) ? r14 : r9; + r14 = (word64)(WC_L64(rsi, 320)); + r10 = (zf7) == (zf8) ? r14 : r10; + r14 = (word64)(WC_L64(rsi, 328)); + r11 = (zf7) == (zf8) ? r14 : r11; + r14 = (word64)(WC_L64(rsi, 336)); + r12 = (zf7) == (zf8) ? r14 : r12; + r14 = (word64)(WC_L64(rsi, 344)); + r13 = (zf7) == (zf8) ? r14 : r13; + zf9 = (byte)r15; + zf10 = 5; + r14 = (word64)(WC_L64(rsi, 384)); + rax = (zf9) == (zf10) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 392)); + rdx = (zf9) == (zf10) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 400)); + r8 = (zf9) == (zf10) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 408)); + r9 = (zf9) == (zf10) ? r14 : r9; + r14 = (word64)(WC_L64(rsi, 416)); + r10 = (zf9) == (zf10) ? r14 : r10; + r14 = (word64)(WC_L64(rsi, 424)); + r11 = (zf9) == (zf10) ? r14 : r11; + r14 = (word64)(WC_L64(rsi, 432)); + r12 = (zf9) == (zf10) ? r14 : r12; + r14 = (word64)(WC_L64(rsi, 440)); + r13 = (zf9) == (zf10) ? r14 : r13; + zf11 = (byte)r15; + zf12 = 6; + r14 = (word64)(WC_L64(rsi, 480)); + rax = (zf11) == (zf12) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 488)); + rdx = (zf11) == (zf12) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 496)); + r8 = (zf11) == (zf12) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 504)); + r9 = (zf11) == (zf12) ? r14 : r9; + r14 = (word64)(WC_L64(rsi, 512)); + r10 = (zf11) == (zf12) ? r14 : r10; + r14 = (word64)(WC_L64(rsi, 520)); + r11 = (zf11) == (zf12) ? r14 : r11; + r14 = (word64)(WC_L64(rsi, 528)); + r12 = (zf11) == (zf12) ? r14 : r12; + r14 = (word64)(WC_L64(rsi, 536)); + r13 = (zf11) == (zf12) ? r14 : r13; + zf13 = (byte)r15; + zf14 = 7; + r14 = (word64)(WC_L64(rsi, 576)); + rax = (zf13) == (zf14) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 584)); + rdx = (zf13) == (zf14) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 592)); + r8 = (zf13) == (zf14) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 600)); + r9 = (zf13) == (zf14) ? r14 : r9; + r14 = (word64)(WC_L64(rsi, 608)); + r10 = (zf13) == (zf14) ? r14 : r10; + r14 = (word64)(WC_L64(rsi, 616)); + r11 = (zf13) == (zf14) ? r14 : r11; + r14 = (word64)(WC_L64(rsi, 624)); + r12 = (zf13) == (zf14) ? r14 : r12; + r14 = (word64)(WC_L64(rsi, 632)); + r13 = (zf13) == (zf14) ? r14 : r13; + zf15 = (byte)r15; + zf16 = 8; + r14 = (word64)(WC_L64(rsi, 672)); + rax = (zf15) == (zf16) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 680)); + rdx = (zf15) == (zf16) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 688)); + r8 = (zf15) == (zf16) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 696)); + r9 = (zf15) == (zf16) ? r14 : r9; + r14 = (word64)(WC_L64(rsi, 704)); + r10 = (zf15) == (zf16) ? r14 : r10; + r14 = (word64)(WC_L64(rsi, 712)); + r11 = (zf15) == (zf16) ? r14 : r11; + r14 = (word64)(WC_L64(rsi, 720)); + r12 = (zf15) == (zf16) ? r14 : r12; + r14 = (word64)(WC_L64(rsi, 728)); + r13 = (zf15) == (zf16) ? r14 : r13; + zf17 = (byte)rcx; + zf18 = 0; + r14 = (word64)(rax); + rax = (sword8)(zf17) < (sword8)(zf18) ? r10 : rax; + r10 = (sword8)(zf17) < (sword8)(zf18) ? r14 : r10; + r14 = (word64)(rdx); + rdx = (sword8)(zf17) < (sword8)(zf18) ? r11 : rdx; + r11 = (sword8)(zf17) < (sword8)(zf18) ? r14 : r11; + r14 = (word64)(r8); + r8 = (sword8)(zf17) < (sword8)(zf18) ? r12 : r8; + r12 = (sword8)(zf17) < (sword8)(zf18) ? r14 : r12; + r14 = (word64)(r9); + r9 = (sword8)(zf17) < (sword8)(zf18) ? r13 : r9; + r13 = (sword8)(zf17) < (sword8)(zf18) ? r14 : r13; + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(r11); + WC_S64(rdi, 48) = (word64)(r12); + WC_S64(rdi, 56) = (word64)(r13); + rax = (word64)(0); + rdx = (word64)(0); + r8 = (word64)(0); + r9 = (word64)(0); + zf19 = (byte)r15; + zf20 = 1; + r14 = (word64)(WC_L64(rsi, 64)); + rax = (zf19) == (zf20) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 72)); + rdx = (zf19) == (zf20) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 80)); + r8 = (zf19) == (zf20) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 88)); + r9 = (zf19) == (zf20) ? r14 : r9; + zf21 = (byte)r15; + zf22 = 2; + r14 = (word64)(WC_L64(rsi, 160)); + rax = (zf21) == (zf22) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 168)); + rdx = (zf21) == (zf22) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 176)); + r8 = (zf21) == (zf22) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 184)); + r9 = (zf21) == (zf22) ? r14 : r9; + zf23 = (byte)r15; + zf24 = 3; + r14 = (word64)(WC_L64(rsi, 256)); + rax = (zf23) == (zf24) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 264)); + rdx = (zf23) == (zf24) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 272)); + r8 = (zf23) == (zf24) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 280)); + r9 = (zf23) == (zf24) ? r14 : r9; + zf25 = (byte)r15; + zf26 = 4; + r14 = (word64)(WC_L64(rsi, 352)); + rax = (zf25) == (zf26) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 360)); + rdx = (zf25) == (zf26) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 368)); + r8 = (zf25) == (zf26) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 376)); + r9 = (zf25) == (zf26) ? r14 : r9; + zf27 = (byte)r15; + zf28 = 5; + r14 = (word64)(WC_L64(rsi, 448)); + rax = (zf27) == (zf28) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 456)); + rdx = (zf27) == (zf28) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 464)); + r8 = (zf27) == (zf28) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 472)); + r9 = (zf27) == (zf28) ? r14 : r9; + zf29 = (byte)r15; + zf30 = 6; + r14 = (word64)(WC_L64(rsi, 544)); + rax = (zf29) == (zf30) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 552)); + rdx = (zf29) == (zf30) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 560)); + r8 = (zf29) == (zf30) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 568)); + r9 = (zf29) == (zf30) ? r14 : r9; + zf31 = (byte)r15; + zf32 = 7; + r14 = (word64)(WC_L64(rsi, 640)); + rax = (zf31) == (zf32) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 648)); + rdx = (zf31) == (zf32) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 656)); + r8 = (zf31) == (zf32) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 664)); + r9 = (zf31) == (zf32) ? r14 : r9; + zf33 = (byte)r15; + zf34 = 8; + r14 = (word64)(WC_L64(rsi, 736)); + rax = (zf33) == (zf34) ? r14 : rax; + r14 = (word64)(WC_L64(rsi, 744)); + rdx = (zf33) == (zf34) ? r14 : rdx; + r14 = (word64)(WC_L64(rsi, 752)); + r8 = (zf33) == (zf34) ? r14 : r8; + r14 = (word64)(WC_L64(rsi, 760)); + r9 = (zf33) == (zf34) ? r14 : r9; + r10 = (word64)(-19); + r11 = (word64)(-1); + r12 = (word64)(-1); + r13 = (word64)(0x7fffffffffffffff); + cf = _subborrow_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, r8, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r9, (unsigned long long*)&r13); + zf35 = (byte)rcx; + zf36 = 0; + rax = (sword8)((byte)rcx) < (sword8)(0) ? r10 : rax; + rdx = (sword8)(zf35) < (sword8)(zf36) ? r11 : rdx; + r8 = (sword8)(zf35) < (sword8)(zf36) ? r12 : r8; + r9 = (sword8)(zf35) < (sword8)(zf36) ? r13 : r9; + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rdx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); +} + +WOLFSSL_LOCAL void fe_mul_x64(fe r, fe a, fe b) +{ + word64 rdi, rsi, rcx, rax, rdx = 0, r8, r9, r10, r11, r12, r13, r14, r15, + rbx; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(size_t)b; + + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r8, rbx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + rbx = (word64)(0x7fffffffffffffff); + rax = (word64)(r11); + rax = (word64)((word64)((sword64)rax >> 63)); + rax = (word64)(rax & 0x13); + r11 = (word64)(r11 & rbx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* Store */ + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); +} + +WOLFSSL_LOCAL void fe_sq_x64(fe r, fe a) +{ + word64 rdi, rsi, rax, rdx = 0, r8, r9, r10, r11, r12, r13, r14, rcx, r15; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Double */ + r14 = (word64)(0); + cf = _addcarry_u64(0, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + r15 = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, r15, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r15 = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r15, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r15 = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r15 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & r15); + r15 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, r15, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + r15 = (word64)(0x7fffffffffffffff); + rax = (word64)(r10); + rax = (word64)((word64)((sword64)rax >> 63)); + rax = (word64)(rax & 0x13); + r10 = (word64)(r10 & r15); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* Store */ + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); +} + +WOLFSSL_LOCAL void fe_sq_n_x64(fe r, const fe a, word64 n) +{ + word64 rdi, rsi, rcx, rax = 0, rdx = 0, r9 = 0, r10 = 0, r11 = 0, r12 = 0, + r13 = 0, r14 = 0, r15 = 0, r8 = 0, rbx = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)n; + +L_fe_sq_n_x64: + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Double */ + r15 = (word64)(0); + cf = _addcarry_u64(0, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r8 = (word64)(rax); + rbx = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rbx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, rbx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r13, rbx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r8, rbx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 1) & 0xff); + if (((byte)rcx) != (0)) { + goto L_fe_sq_n_x64; + } +} + +WOLFSSL_LOCAL void fe_mul121666_x64(fe r, fe a) +{ + word64 rdi, rsi, rax, rdx = 0, r10, r8, r9, r11, r12, rcx; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + /* Multiply by 121666 */ + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + r8 = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + rcx = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + r12 = (word64)((r12 << 1) | (r11 >> 63)); + r11 = (word64)(r11 & rcx); + rax = (word64)(0x13); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); +} + +WOLFSSL_LOCAL void fe_invert_x64(fe r, const fe a) +{ + word64 rdi, rsi, rsp, rdx; + XALIGNED(32) WC_X64I_SLOT stk[20]; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 144); + /* Invert */ + WC_S64(rsp, 128) = (word64)(rdi); + WC_S64(rsp, 136) = (word64)(rsi); + rdi = (word64)(rsp); + rsi = (word64)(WC_L64(rsp, 136)); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(WC_L64(rsp, 136)); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(4); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(9); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(0x13); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(9); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(0x31); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(0x63); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(0x31); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(4); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(WC_L64(rsp, 128)); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rsi = (word64)(WC_L64(rsp, 136)); + rdi = (word64)(WC_L64(rsp, 128)); +} + +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +XALIGNED(32) static const word64 L_curve25519_base_x64_x2[] WC_X64I_UNUSED = { + 0x5cae469cdd684efbULL, 0x8f3f5ced1e350b5cULL, + 0xd9750c687d157114ULL, 0x20d342d51873f1b7ULL, +}; + +WOLFSSL_LOCAL int curve25519_base_x64(byte* r, byte* n) +{ + word64 rdi, rsi, rsp, r15, rcx, r8, r9, r10, rbp, rbx = 0, r11 = 0, + r12 = 0, r13 = 0, r14 = 0, rax = 0, rdx = 0; + XALIGNED(32) WC_X64I_SLOT stk[24]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)n; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 168); + r15 = (word64)(0); + WC_S64(rsp, 160) = (word64)(rdi); + /* Set base point x */ + WC_S64(rdi, 0) = (word64)(9); + WC_S64(rdi, 8) = (word64)(0); + WC_S64(rdi, 16) = (word64)(0); + WC_S64(rdi, 24) = (word64)(0); + /* Set one */ + WC_S64(rsp, 0) = (word64)(1); + WC_S64(rsp, 8) = (word64)(0); + WC_S64(rsp, 16) = (word64)(0); + WC_S64(rsp, 24) = (word64)(0); + rcx = (word64)(WC_L64(L_curve25519_base_x64_x2, 0)); + r8 = (word64)(WC_L64(L_curve25519_base_x64_x2, 8)); + r9 = (word64)(WC_L64(L_curve25519_base_x64_x2, 16)); + r10 = (word64)(WC_L64(L_curve25519_base_x64_x2, 24)); + /* Set one */ + WC_S64(rsp, 32) = (word64)(1); + WC_S64(rsp, 40) = (word64)(0); + WC_S64(rsp, 48) = (word64)(0); + WC_S64(rsp, 56) = (word64)(0); + WC_S64(rsp, 64) = (word64)(rcx); + WC_S64(rsp, 72) = (word64)(r8); + WC_S64(rsp, 80) = (word64)(r9); + WC_S64(rsp, 88) = (word64)(r10); + rbp = (word64)(0xfd); +L_curve25519_base_x64_bits: + r8 = (word64)(rbp); + rcx = (word64)(rbp); + rcx = (word64)(rcx & 0x3f); + r8 = (word64)(r8 >> 6); + rbx = (word64)(WC_L64(rsi, r8 * 8)); + rbx = (word64)(rbx >> ((byte)rcx & 63)); + rbx = (word64)(rbx & 1); + r15 = (word64)(r15 ^ rbx); + r15 = (word64)(0 - r15); + /* Conditional Swap */ + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rdi, 24)); + r11 = (word64)(WC_L64(rsp, 0)); + r12 = (word64)(WC_L64(rsp, 8)); + r13 = (word64)(WC_L64(rsp, 16)); + r14 = (word64)(WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ WC_L64(rsp, 64)); + r8 = (word64)(r8 ^ WC_L64(rsp, 72)); + r9 = (word64)(r9 ^ WC_L64(rsp, 80)); + r10 = (word64)(r10 ^ WC_L64(rsp, 88)); + r11 = (word64)(r11 ^ WC_L64(rsp, 32)); + r12 = (word64)(r12 ^ WC_L64(rsp, 40)); + r13 = (word64)(r13 ^ WC_L64(rsp, 48)); + r14 = (word64)(r14 ^ WC_L64(rsp, 56)); + rcx = (word64)(rcx & r15); + r8 = (word64)(r8 & r15); + r9 = (word64)(r9 & r15); + r10 = (word64)(r10 & r15); + r11 = (word64)(r11 & r15); + r12 = (word64)(r12 & r15); + r13 = (word64)(r13 & r15); + r14 = (word64)(r14 & r15); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) ^ rcx); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) ^ r8); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) ^ r9); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) ^ r10); + WC_S64(rsp, 0) = (word64)(WC_L64(rsp, 0) ^ r11); + WC_S64(rsp, 8) = (word64)(WC_L64(rsp, 8) ^ r12); + WC_S64(rsp, 16) = (word64)(WC_L64(rsp, 16) ^ r13); + WC_S64(rsp, 24) = (word64)(WC_L64(rsp, 24) ^ r14); + WC_S64(rsp, 64) = (word64)(WC_L64(rsp, 64) ^ rcx); + WC_S64(rsp, 72) = (word64)(WC_L64(rsp, 72) ^ r8); + WC_S64(rsp, 80) = (word64)(WC_L64(rsp, 80) ^ r9); + WC_S64(rsp, 88) = (word64)(WC_L64(rsp, 88) ^ r10); + WC_S64(rsp, 32) = (word64)(WC_L64(rsp, 32) ^ r11); + WC_S64(rsp, 40) = (word64)(WC_L64(rsp, 40) ^ r12); + WC_S64(rsp, 48) = (word64)(WC_L64(rsp, 48) ^ r13); + WC_S64(rsp, 56) = (word64)(WC_L64(rsp, 56) ^ r14); + r15 = (word64)(rbx); + /* Add-Sub */ + /* Add */ + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rdi, 24)); + r11 = (word64)(rcx); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 0), (unsigned long long*)&rcx); + r12 = (word64)(r8); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 8), (unsigned long long*)&r8); + r13 = (word64)(r9); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 16), (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 24), (unsigned long long*)&r10); + rbx = (word64)(0); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r10 >> 63)); + rbx = (word64)(rbx * 0x13); + r10 = (word64)(r10 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* Sub */ + cf = _subborrow_u64(0, r11, WC_L64(rsp, 0), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, WC_L64(rsp, 8), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, WC_L64(rsp, 16), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rsp, 24), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, rbx, rbx, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r14 >> 63)); + rbx = (word64)(rbx * -19); + r14 = (word64)(r14 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r11, rbx, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + WC_S64(rsp, 128) = (word64)(r11); + WC_S64(rsp, 136) = (word64)(r12); + WC_S64(rsp, 144) = (word64)(r13); + WC_S64(rsp, 152) = (word64)(r14); + /* Add-Sub */ + /* Add */ + rcx = (word64)(WC_L64(rsp, 64)); + r8 = (word64)(WC_L64(rsp, 72)); + r9 = (word64)(WC_L64(rsp, 80)); + r10 = (word64)(WC_L64(rsp, 88)); + r11 = (word64)(rcx); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 32), (unsigned long long*)&rcx); + r12 = (word64)(r8); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 40), (unsigned long long*)&r8); + r13 = (word64)(r9); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 48), (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 56), (unsigned long long*)&r10); + rbx = (word64)(0); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r10 >> 63)); + rbx = (word64)(rbx * 0x13); + r10 = (word64)(r10 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* Sub */ + cf = _subborrow_u64(0, r11, WC_L64(rsp, 32), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, WC_L64(rsp, 40), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, WC_L64(rsp, 48), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rsp, 56), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, rbx, rbx, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r14 >> 63)); + rbx = (word64)(rbx * -19); + r14 = (word64)(r14 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r11, rbx, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 32) = (word64)(rcx); + WC_S64(rsp, 40) = (word64)(r8); + WC_S64(rsp, 48) = (word64)(r9); + WC_S64(rsp, 56) = (word64)(r10); + WC_S64(rsp, 96) = (word64)(r11); + WC_S64(rsp, 104) = (word64)(r12); + WC_S64(rsp, 112) = (word64)(r13); + WC_S64(rsp, 120) = (word64)(r14); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 32)); + rcx = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 32)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 32)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 32) = (word64)(rcx); + WC_S64(rsp, 40) = (word64)(r8); + WC_S64(rsp, 48) = (word64)(r9); + WC_S64(rsp, 56) = (word64)(r10); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 96)); + rcx = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 96)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 104)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 104)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 96)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 104)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 112)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 120)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 112)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 120)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 120)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 0) = (word64)(rcx); + WC_S64(rsp, 8) = (word64)(r8); + WC_S64(rsp, 16) = (word64)(r9); + WC_S64(rsp, 24) = (word64)(r10); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Double */ + r14 = (word64)(0); + cf = _addcarry_u64(0, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbx = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rbx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rbx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 96) = (word64)(rcx); + WC_S64(rsp, 104) = (word64)(r8); + WC_S64(rsp, 112) = (word64)(r9); + WC_S64(rsp, 120) = (word64)(r10); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Double */ + r14 = (word64)(0); + cf = _addcarry_u64(0, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbx = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rbx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rbx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 128) = (word64)(rcx); + WC_S64(rsp, 136) = (word64)(r8); + WC_S64(rsp, 144) = (word64)(r9); + WC_S64(rsp, 152) = (word64)(r10); + /* Add-Sub */ + /* Add */ + rcx = (word64)(WC_L64(rsp, 0)); + r8 = (word64)(WC_L64(rsp, 8)); + r9 = (word64)(WC_L64(rsp, 16)); + r10 = (word64)(WC_L64(rsp, 24)); + r11 = (word64)(rcx); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 32), (unsigned long long*)&rcx); + r12 = (word64)(r8); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 40), (unsigned long long*)&r8); + r13 = (word64)(r9); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 48), (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 56), (unsigned long long*)&r10); + rbx = (word64)(0); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r10 >> 63)); + rbx = (word64)(rbx * 0x13); + r10 = (word64)(r10 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* Sub */ + cf = _subborrow_u64(0, r11, WC_L64(rsp, 32), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, WC_L64(rsp, 40), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, WC_L64(rsp, 48), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rsp, 56), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, rbx, rbx, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r14 >> 63)); + rbx = (word64)(rbx * -19); + r14 = (word64)(r14 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r11, rbx, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 64) = (word64)(rcx); + WC_S64(rsp, 72) = (word64)(r8); + WC_S64(rsp, 80) = (word64)(r9); + WC_S64(rsp, 88) = (word64)(r10); + WC_S64(rsp, 32) = (word64)(r11); + WC_S64(rsp, 40) = (word64)(r12); + WC_S64(rsp, 48) = (word64)(r13); + WC_S64(rsp, 56) = (word64)(r14); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + rcx = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + /* Sub */ + rcx = (word64)(WC_L64(rsp, 128)); + r8 = (word64)(WC_L64(rsp, 136)); + r9 = (word64)(WC_L64(rsp, 144)); + r10 = (word64)(WC_L64(rsp, 152)); + cf = _subborrow_u64(0, rcx, WC_L64(rsp, 96), (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 104), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rsp, 112), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rsp, 120), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, rbx, rbx, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r10 >> 63)); + rbx = (word64)(rbx * -19); + r10 = (word64)(r10 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 128) = (word64)(rcx); + WC_S64(rsp, 136) = (word64)(r8); + WC_S64(rsp, 144) = (word64)(r9); + WC_S64(rsp, 152) = (word64)(r10); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsp, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Double */ + r14 = (word64)(0); + cf = _addcarry_u64(0, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbx = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rbx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsp, 48)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsp, 56)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rbx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 32) = (word64)(rcx); + WC_S64(rsp, 40) = (word64)(r8); + WC_S64(rsp, 48) = (word64)(r9); + WC_S64(rsp, 56) = (word64)(r10); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsp, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 72)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsp, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 80)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsp, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 88)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsp, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 80)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsp, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 88)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsp, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 88)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Double */ + r14 = (word64)(0); + cf = _addcarry_u64(0, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsp, 64)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbx = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsp, 72)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rbx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsp, 80)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsp, 88)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rbx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 64) = (word64)(rcx); + WC_S64(rsp, 72) = (word64)(r8); + WC_S64(rsp, 80) = (word64)(r9); + WC_S64(rsp, 88) = (word64)(r10); + /* Multiply by 121666 */ + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r9 = (word64)(0); + rcx = (word64)(rax); + r8 = (word64)(rdx); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r11 = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 96), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 104), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 112), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 120), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + r12 = (word64)((r12 << 1) | (r10 >> 63)); + r10 = (word64)(r10 & r11); + rax = (word64)(0x13); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 96) = (word64)(rcx); + WC_S64(rsp, 104) = (word64)(r8); + WC_S64(rsp, 112) = (word64)(r9); + WC_S64(rsp, 120) = (word64)(r10); + /* Multiply by 9 */ + rax = (word64)(9); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 32)); + r9 = (word64)(0); + rcx = (word64)(rax); + r8 = (word64)(rdx); + rax = (word64)(9); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + rax = (word64)(9); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + rax = (word64)(9); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + r11 = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + r12 = (word64)((r12 << 1) | (r10 >> 63)); + r10 = (word64)(r10 & r11); + rax = (word64)(0x13); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 32) = (word64)(rcx); + WC_S64(rsp, 40) = (word64)(r8); + WC_S64(rsp, 48) = (word64)(r9); + WC_S64(rsp, 56) = (word64)(r10); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + rcx = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 0) = (word64)(rcx); + WC_S64(rsp, 8) = (word64)(r8); + WC_S64(rsp, 16) = (word64)(r9); + WC_S64(rsp, 24) = (word64)(r10); + rbp = (word64)(rbp - 1); + if ((sword64)(rbp) >= (sword64)(3)) { + goto L_curve25519_base_x64_bits; + } + r15 = (word64)(0 - r15); + /* Conditional Swap */ + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rdi, 24)); + r11 = (word64)(WC_L64(rsp, 0)); + r12 = (word64)(WC_L64(rsp, 8)); + r13 = (word64)(WC_L64(rsp, 16)); + r14 = (word64)(WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ WC_L64(rsp, 64)); + r8 = (word64)(r8 ^ WC_L64(rsp, 72)); + r9 = (word64)(r9 ^ WC_L64(rsp, 80)); + r10 = (word64)(r10 ^ WC_L64(rsp, 88)); + r11 = (word64)(r11 ^ WC_L64(rsp, 32)); + r12 = (word64)(r12 ^ WC_L64(rsp, 40)); + r13 = (word64)(r13 ^ WC_L64(rsp, 48)); + r14 = (word64)(r14 ^ WC_L64(rsp, 56)); + rcx = (word64)(rcx & r15); + r8 = (word64)(r8 & r15); + r9 = (word64)(r9 & r15); + r10 = (word64)(r10 & r15); + r11 = (word64)(r11 & r15); + r12 = (word64)(r12 & r15); + r13 = (word64)(r13 & r15); + r14 = (word64)(r14 & r15); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) ^ rcx); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) ^ r8); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) ^ r9); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) ^ r10); + WC_S64(rsp, 0) = (word64)(WC_L64(rsp, 0) ^ r11); + WC_S64(rsp, 8) = (word64)(WC_L64(rsp, 8) ^ r12); + WC_S64(rsp, 16) = (word64)(WC_L64(rsp, 16) ^ r13); + WC_S64(rsp, 24) = (word64)(WC_L64(rsp, 24) ^ r14); + WC_S64(rsp, 64) = (word64)(WC_L64(rsp, 64) ^ rcx); + WC_S64(rsp, 72) = (word64)(WC_L64(rsp, 72) ^ r8); + WC_S64(rsp, 80) = (word64)(WC_L64(rsp, 80) ^ r9); + WC_S64(rsp, 88) = (word64)(WC_L64(rsp, 88) ^ r10); + WC_S64(rsp, 32) = (word64)(WC_L64(rsp, 32) ^ r11); + WC_S64(rsp, 40) = (word64)(WC_L64(rsp, 40) ^ r12); + WC_S64(rsp, 48) = (word64)(WC_L64(rsp, 48) ^ r13); + WC_S64(rsp, 56) = (word64)(WC_L64(rsp, 56) ^ r14); +L_curve25519_base_x64_3: + /* Add-Sub */ + /* Add */ + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rdi, 24)); + r11 = (word64)(rcx); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 0), (unsigned long long*)&rcx); + r12 = (word64)(r8); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 8), (unsigned long long*)&r8); + r13 = (word64)(r9); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 16), (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 24), (unsigned long long*)&r10); + rbx = (word64)(0); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r10 >> 63)); + rbx = (word64)(rbx * 0x13); + r10 = (word64)(r10 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* Sub */ + cf = _subborrow_u64(0, r11, WC_L64(rsp, 0), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, WC_L64(rsp, 8), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, WC_L64(rsp, 16), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rsp, 24), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, rbx, rbx, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r14 >> 63)); + rbx = (word64)(rbx * -19); + r14 = (word64)(r14 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r11, rbx, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + WC_S64(rsp, 128) = (word64)(r11); + WC_S64(rsp, 136) = (word64)(r12); + WC_S64(rsp, 144) = (word64)(r13); + WC_S64(rsp, 152) = (word64)(r14); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Double */ + r14 = (word64)(0); + cf = _addcarry_u64(0, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbx = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rbx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rbx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 96) = (word64)(rcx); + WC_S64(rsp, 104) = (word64)(r8); + WC_S64(rsp, 112) = (word64)(r9); + WC_S64(rsp, 120) = (word64)(r10); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Double */ + r14 = (word64)(0); + cf = _addcarry_u64(0, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbx = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rbx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rbx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 128) = (word64)(rcx); + WC_S64(rsp, 136) = (word64)(r8); + WC_S64(rsp, 144) = (word64)(r9); + WC_S64(rsp, 152) = (word64)(r10); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + rcx = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + /* Sub */ + rcx = (word64)(WC_L64(rsp, 128)); + r8 = (word64)(WC_L64(rsp, 136)); + r9 = (word64)(WC_L64(rsp, 144)); + r10 = (word64)(WC_L64(rsp, 152)); + cf = _subborrow_u64(0, rcx, WC_L64(rsp, 96), (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 104), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rsp, 112), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rsp, 120), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, rbx, rbx, (unsigned long long*)&rbx); + rbx = (word64)((rbx << 1) | (r10 >> 63)); + rbx = (word64)(rbx * -19); + r10 = (word64)(r10 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 128) = (word64)(rcx); + WC_S64(rsp, 136) = (word64)(r8); + WC_S64(rsp, 144) = (word64)(r9); + WC_S64(rsp, 152) = (word64)(r10); + /* Multiply by 121666 */ + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r9 = (word64)(0); + rcx = (word64)(rax); + r8 = (word64)(rdx); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r11 = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 96), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 104), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 112), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 120), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + r12 = (word64)((r12 << 1) | (r10 >> 63)); + r10 = (word64)(r10 & r11); + rax = (word64)(0x13); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 96) = (word64)(rcx); + WC_S64(rsp, 104) = (word64)(r8); + WC_S64(rsp, 112) = (word64)(r9); + WC_S64(rsp, 120) = (word64)(r10); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + rcx = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + /* Store */ + WC_S64(rsp, 0) = (word64)(rcx); + WC_S64(rsp, 8) = (word64)(r8); + WC_S64(rsp, 16) = (word64)(r9); + WC_S64(rsp, 24) = (word64)(r10); + rbp = (word64)(rbp - 1); + if ((sword64)(rbp) >= (sword64)(0)) { + goto L_curve25519_base_x64_3; + } + /* Invert */ + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 96); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(4); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(9); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 128); + rsi = (word64)(rsp + 96); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 128); + rsi = (word64)(rsp + 128); + rdx = (word64)(0x13); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 128); + rdx = (word64)(rsp + 96); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(9); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(0x31); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 128); + rsi = (word64)(rsp + 96); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 128); + rsi = (word64)(rsp + 128); + rdx = (word64)(0x63); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 128); + rdx = (word64)(rsp + 96); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(0x31); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(4); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(WC_L64(rsp, 160)); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + rcx = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & rbx); + rbx = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, rbx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + rbx = (word64)(0x7fffffffffffffff); + rax = (word64)(r10); + rax = (word64)((word64)((sword64)rax >> 63)); + rax = (word64)(rax & 0x13); + r10 = (word64)(r10 & rbx); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + rax = (word64)(0x7fffffffffffffff); + rdx = (word64)(rcx); + cf = _addcarry_u64(0, rdx, 0x13, (unsigned long long*)&rdx); + rdx = (word64)(r8); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rdx = (word64)(r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rdx = (word64)(r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rdx = (word64)((word64)((sword64)rdx >> 63)); + rdx = (word64)(rdx & 0x13); + r10 = (word64)(r10 & rax); + cf = _addcarry_u64(0, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r10 = (word64)(r10 & rax); + /* Store */ + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + rax = (word64)(0); + return (int)(word32)rax; +} + +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ +WOLFSSL_LOCAL int curve25519_x64(byte* r, byte* n, byte* a) +{ + word64 rdi, rsi, r8, rsp, rbx, rcx, r9, r10, r11, rbp = 0, r12 = 0, + r13 = 0, r14 = 0, r15 = 0, rax = 0, rdx = 0; + XALIGNED(32) WC_X64I_SLOT stk[24]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)n; + r8 = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 176); + rbx = (word64)(0); + WC_S64(rsp, 168) = (word64)(rdi); + /* Set one */ + WC_S64(rdi, 0) = (word64)(1); + WC_S64(rdi, 8) = (word64)(0); + WC_S64(rdi, 16) = (word64)(0); + WC_S64(rdi, 24) = (word64)(0); + /* Set zero */ + WC_S64(rsp, 0) = (word64)(0); + WC_S64(rsp, 8) = (word64)(0); + WC_S64(rsp, 16) = (word64)(0); + WC_S64(rsp, 24) = (word64)(0); + /* Set one */ + WC_S64(rsp, 32) = (word64)(1); + WC_S64(rsp, 40) = (word64)(0); + WC_S64(rsp, 48) = (word64)(0); + WC_S64(rsp, 56) = (word64)(0); + /* Copy */ + rcx = (word64)(WC_L64(r8, 0)); + r9 = (word64)(WC_L64(r8, 8)); + r10 = (word64)(WC_L64(r8, 16)); + r11 = (word64)(WC_L64(r8, 24)); + WC_S64(rsp, 64) = (word64)(rcx); + WC_S64(rsp, 72) = (word64)(r9); + WC_S64(rsp, 80) = (word64)(r10); + WC_S64(rsp, 88) = (word64)(r11); + r9 = (word64)(0xfe); +L_curve25519_x64_bits: + WC_S64(rsp, 160) = (word64)(r9); + rcx = (word64)(r9); + rcx = (word64)(rcx & 0x3f); + r9 = (word64)(r9 >> 6); + rbp = (word64)(WC_L64(rsi, r9 * 8)); + rbp = (word64)(rbp >> ((byte)rcx & 63)); + rbp = (word64)(rbp & 1); + rbx = (word64)(rbx ^ rbp); + rbx = (word64)(0 - rbx); + /* Conditional Swap */ + rcx = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rsp, 0)); + r13 = (word64)(WC_L64(rsp, 8)); + r14 = (word64)(WC_L64(rsp, 16)); + r15 = (word64)(WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ WC_L64(rsp, 64)); + r9 = (word64)(r9 ^ WC_L64(rsp, 72)); + r10 = (word64)(r10 ^ WC_L64(rsp, 80)); + r11 = (word64)(r11 ^ WC_L64(rsp, 88)); + r12 = (word64)(r12 ^ WC_L64(rsp, 32)); + r13 = (word64)(r13 ^ WC_L64(rsp, 40)); + r14 = (word64)(r14 ^ WC_L64(rsp, 48)); + r15 = (word64)(r15 ^ WC_L64(rsp, 56)); + rcx = (word64)(rcx & rbx); + r9 = (word64)(r9 & rbx); + r10 = (word64)(r10 & rbx); + r11 = (word64)(r11 & rbx); + r12 = (word64)(r12 & rbx); + r13 = (word64)(r13 & rbx); + r14 = (word64)(r14 & rbx); + r15 = (word64)(r15 & rbx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) ^ rcx); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) ^ r9); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) ^ r10); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) ^ r11); + WC_S64(rsp, 0) = (word64)(WC_L64(rsp, 0) ^ r12); + WC_S64(rsp, 8) = (word64)(WC_L64(rsp, 8) ^ r13); + WC_S64(rsp, 16) = (word64)(WC_L64(rsp, 16) ^ r14); + WC_S64(rsp, 24) = (word64)(WC_L64(rsp, 24) ^ r15); + WC_S64(rsp, 64) = (word64)(WC_L64(rsp, 64) ^ rcx); + WC_S64(rsp, 72) = (word64)(WC_L64(rsp, 72) ^ r9); + WC_S64(rsp, 80) = (word64)(WC_L64(rsp, 80) ^ r10); + WC_S64(rsp, 88) = (word64)(WC_L64(rsp, 88) ^ r11); + WC_S64(rsp, 32) = (word64)(WC_L64(rsp, 32) ^ r12); + WC_S64(rsp, 40) = (word64)(WC_L64(rsp, 40) ^ r13); + WC_S64(rsp, 48) = (word64)(WC_L64(rsp, 48) ^ r14); + WC_S64(rsp, 56) = (word64)(WC_L64(rsp, 56) ^ r15); + rbx = (word64)(rbp); + /* Add-Sub */ + /* Add */ + rcx = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(rcx); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 0), (unsigned long long*)&rcx); + r13 = (word64)(r9); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 8), (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 16), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsp, 24), (unsigned long long*)&r11); + rbp = (word64)(0); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r11 >> 63)); + rbp = (word64)(rbp * 0x13); + r11 = (word64)(r11 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* Sub */ + cf = _subborrow_u64(0, r12, WC_L64(rsp, 0), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, WC_L64(rsp, 8), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rsp, 16), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsp, 24), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbp, rbp, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r15 >> 63)); + rbp = (word64)(rbp * -19); + r15 = (word64)(r15 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r12, rbp, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rsp, 128) = (word64)(r12); + WC_S64(rsp, 136) = (word64)(r13); + WC_S64(rsp, 144) = (word64)(r14); + WC_S64(rsp, 152) = (word64)(r15); + /* Add-Sub */ + /* Add */ + rcx = (word64)(WC_L64(rsp, 64)); + r9 = (word64)(WC_L64(rsp, 72)); + r10 = (word64)(WC_L64(rsp, 80)); + r11 = (word64)(WC_L64(rsp, 88)); + r12 = (word64)(rcx); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 32), (unsigned long long*)&rcx); + r13 = (word64)(r9); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 40), (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 48), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsp, 56), (unsigned long long*)&r11); + rbp = (word64)(0); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r11 >> 63)); + rbp = (word64)(rbp * 0x13); + r11 = (word64)(r11 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* Sub */ + cf = _subborrow_u64(0, r12, WC_L64(rsp, 32), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, WC_L64(rsp, 40), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rsp, 48), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsp, 56), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbp, rbp, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r15 >> 63)); + rbp = (word64)(rbp * -19); + r15 = (word64)(r15 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r12, rbp, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + WC_S64(rsp, 32) = (word64)(rcx); + WC_S64(rsp, 40) = (word64)(r9); + WC_S64(rsp, 48) = (word64)(r10); + WC_S64(rsp, 56) = (word64)(r11); + WC_S64(rsp, 96) = (word64)(r12); + WC_S64(rsp, 104) = (word64)(r13); + WC_S64(rsp, 112) = (word64)(r14); + WC_S64(rsp, 120) = (word64)(r15); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 32)); + rcx = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 32)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 32)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 32) = (word64)(rcx); + WC_S64(rsp, 40) = (word64)(r9); + WC_S64(rsp, 48) = (word64)(r10); + WC_S64(rsp, 56) = (word64)(r11); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 96)); + rcx = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 96)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 104)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 104)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 96)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 104)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 112)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 120)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 104)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 112)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 120)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 112)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 120)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 120)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 0) = (word64)(rcx); + WC_S64(rsp, 8) = (word64)(r9); + WC_S64(rsp, 16) = (word64)(r10); + WC_S64(rsp, 24) = (word64)(r11); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Double */ + r15 = (word64)(0); + cf = _addcarry_u64(0, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbp = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, rbp, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r13, rbp, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 96) = (word64)(rcx); + WC_S64(rsp, 104) = (word64)(r9); + WC_S64(rsp, 112) = (word64)(r10); + WC_S64(rsp, 120) = (word64)(r11); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Double */ + r15 = (word64)(0); + cf = _addcarry_u64(0, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbp = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, rbp, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r13, rbp, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 128) = (word64)(rcx); + WC_S64(rsp, 136) = (word64)(r9); + WC_S64(rsp, 144) = (word64)(r10); + WC_S64(rsp, 152) = (word64)(r11); + /* Add-Sub */ + /* Add */ + rcx = (word64)(WC_L64(rsp, 0)); + r9 = (word64)(WC_L64(rsp, 8)); + r10 = (word64)(WC_L64(rsp, 16)); + r11 = (word64)(WC_L64(rsp, 24)); + r12 = (word64)(rcx); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 32), (unsigned long long*)&rcx); + r13 = (word64)(r9); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 40), (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 48), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsp, 56), (unsigned long long*)&r11); + rbp = (word64)(0); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r11 >> 63)); + rbp = (word64)(rbp * 0x13); + r11 = (word64)(r11 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* Sub */ + cf = _subborrow_u64(0, r12, WC_L64(rsp, 32), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, WC_L64(rsp, 40), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rsp, 48), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsp, 56), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbp, rbp, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r15 >> 63)); + rbp = (word64)(rbp * -19); + r15 = (word64)(r15 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r12, rbp, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + WC_S64(rsp, 64) = (word64)(rcx); + WC_S64(rsp, 72) = (word64)(r9); + WC_S64(rsp, 80) = (word64)(r10); + WC_S64(rsp, 88) = (word64)(r11); + WC_S64(rsp, 32) = (word64)(r12); + WC_S64(rsp, 40) = (word64)(r13); + WC_S64(rsp, 48) = (word64)(r14); + WC_S64(rsp, 56) = (word64)(r15); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + rcx = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + /* Sub */ + rcx = (word64)(WC_L64(rsp, 128)); + r9 = (word64)(WC_L64(rsp, 136)); + r10 = (word64)(WC_L64(rsp, 144)); + r11 = (word64)(WC_L64(rsp, 152)); + cf = _subborrow_u64(0, rcx, WC_L64(rsp, 96), (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r9, WC_L64(rsp, 104), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rsp, 112), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, WC_L64(rsp, 120), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, rbp, rbp, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r11 >> 63)); + rbp = (word64)(rbp * -19); + r11 = (word64)(r11 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rsp, 128) = (word64)(rcx); + WC_S64(rsp, 136) = (word64)(r9); + WC_S64(rsp, 144) = (word64)(r10); + WC_S64(rsp, 152) = (word64)(r11); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 40)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 48)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsp, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 56)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Double */ + r15 = (word64)(0); + cf = _addcarry_u64(0, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbp = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsp, 48)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, rbp, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsp, 56)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r13, rbp, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 32) = (word64)(rcx); + WC_S64(rsp, 40) = (word64)(r9); + WC_S64(rsp, 48) = (word64)(r10); + WC_S64(rsp, 56) = (word64)(r11); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsp, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 72)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsp, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 80)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsp, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 88)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsp, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 80)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsp, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 88)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsp, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 88)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Double */ + r15 = (word64)(0); + cf = _addcarry_u64(0, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsp, 64)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbp = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsp, 72)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsp, 80)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, rbp, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsp, 88)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r13, rbp, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + rbp = (word64)(0x7fffffffffffffff); + rax = (word64)(r11); + rax = (word64)((word64)((sword64)rax >> 63)); + rax = (word64)(rax & 0x13); + r11 = (word64)(r11 & rbp); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 64) = (word64)(rcx); + WC_S64(rsp, 72) = (word64)(r9); + WC_S64(rsp, 80) = (word64)(r10); + WC_S64(rsp, 88) = (word64)(r11); + /* Multiply by 121666 */ + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r10 = (word64)(0); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r12 = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 96), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 104), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 112), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, WC_L64(rsp, 120), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + r13 = (word64)((r13 << 1) | (r11 >> 63)); + r11 = (word64)(r11 & r12); + rax = (word64)(0x13); + WC_X64I_MUL128(rax, rdx, rax, r13); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rsp, 96) = (word64)(rcx); + WC_S64(rsp, 104) = (word64)(r9); + WC_S64(rsp, 112) = (word64)(r10); + WC_S64(rsp, 120) = (word64)(r11); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + rcx = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 32) = (word64)(rcx); + WC_S64(rsp, 40) = (word64)(r9); + WC_S64(rsp, 48) = (word64)(r10); + WC_S64(rsp, 56) = (word64)(r11); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + rcx = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 0) = (word64)(rcx); + WC_S64(rsp, 8) = (word64)(r9); + WC_S64(rsp, 16) = (word64)(r10); + WC_S64(rsp, 24) = (word64)(r11); + r9 = (word64)(WC_L64(rsp, 160)); + r9 = (word64)(r9 - 1); + if ((sword64)(r9) >= (sword64)(3)) { + goto L_curve25519_x64_bits; + } + WC_S64(rsp, 160) = (word64)(2); + rbx = (word64)(0 - rbx); + /* Conditional Swap */ + rcx = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rsp, 0)); + r13 = (word64)(WC_L64(rsp, 8)); + r14 = (word64)(WC_L64(rsp, 16)); + r15 = (word64)(WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ WC_L64(rsp, 64)); + r9 = (word64)(r9 ^ WC_L64(rsp, 72)); + r10 = (word64)(r10 ^ WC_L64(rsp, 80)); + r11 = (word64)(r11 ^ WC_L64(rsp, 88)); + r12 = (word64)(r12 ^ WC_L64(rsp, 32)); + r13 = (word64)(r13 ^ WC_L64(rsp, 40)); + r14 = (word64)(r14 ^ WC_L64(rsp, 48)); + r15 = (word64)(r15 ^ WC_L64(rsp, 56)); + rcx = (word64)(rcx & rbx); + r9 = (word64)(r9 & rbx); + r10 = (word64)(r10 & rbx); + r11 = (word64)(r11 & rbx); + r12 = (word64)(r12 & rbx); + r13 = (word64)(r13 & rbx); + r14 = (word64)(r14 & rbx); + r15 = (word64)(r15 & rbx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) ^ rcx); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) ^ r9); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) ^ r10); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) ^ r11); + WC_S64(rsp, 0) = (word64)(WC_L64(rsp, 0) ^ r12); + WC_S64(rsp, 8) = (word64)(WC_L64(rsp, 8) ^ r13); + WC_S64(rsp, 16) = (word64)(WC_L64(rsp, 16) ^ r14); + WC_S64(rsp, 24) = (word64)(WC_L64(rsp, 24) ^ r15); + WC_S64(rsp, 64) = (word64)(WC_L64(rsp, 64) ^ rcx); + WC_S64(rsp, 72) = (word64)(WC_L64(rsp, 72) ^ r9); + WC_S64(rsp, 80) = (word64)(WC_L64(rsp, 80) ^ r10); + WC_S64(rsp, 88) = (word64)(WC_L64(rsp, 88) ^ r11); + WC_S64(rsp, 32) = (word64)(WC_L64(rsp, 32) ^ r12); + WC_S64(rsp, 40) = (word64)(WC_L64(rsp, 40) ^ r13); + WC_S64(rsp, 48) = (word64)(WC_L64(rsp, 48) ^ r14); + WC_S64(rsp, 56) = (word64)(WC_L64(rsp, 56) ^ r15); +L_curve25519_x64_3: + /* Add-Sub */ + /* Add */ + rcx = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(rcx); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 0), (unsigned long long*)&rcx); + r13 = (word64)(r9); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 8), (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 16), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsp, 24), (unsigned long long*)&r11); + rbp = (word64)(0); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r11 >> 63)); + rbp = (word64)(rbp * 0x13); + r11 = (word64)(r11 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* Sub */ + cf = _subborrow_u64(0, r12, WC_L64(rsp, 0), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, WC_L64(rsp, 8), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rsp, 16), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsp, 24), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbp, rbp, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r15 >> 63)); + rbp = (word64)(rbp * -19); + r15 = (word64)(r15 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r12, rbp, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rsp, 128) = (word64)(r12); + WC_S64(rsp, 136) = (word64)(r13); + WC_S64(rsp, 144) = (word64)(r14); + WC_S64(rsp, 152) = (word64)(r15); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Double */ + r15 = (word64)(0); + cf = _addcarry_u64(0, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsp, 128)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbp = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsp, 136)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsp, 144)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, rbp, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsp, 152)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r13, rbp, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 96) = (word64)(rcx); + WC_S64(rsp, 104) = (word64)(r9); + WC_S64(rsp, 112) = (word64)(r10); + WC_S64(rsp, 120) = (word64)(r11); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Double */ + r15 = (word64)(0); + cf = _addcarry_u64(0, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + rbp = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, rbp, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r13, rbp, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 128) = (word64)(rcx); + WC_S64(rsp, 136) = (word64)(r9); + WC_S64(rsp, 144) = (word64)(r10); + WC_S64(rsp, 152) = (word64)(r11); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + rcx = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + /* Sub */ + rcx = (word64)(WC_L64(rsp, 128)); + r9 = (word64)(WC_L64(rsp, 136)); + r10 = (word64)(WC_L64(rsp, 144)); + r11 = (word64)(WC_L64(rsp, 152)); + cf = _subborrow_u64(0, rcx, WC_L64(rsp, 96), (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r9, WC_L64(rsp, 104), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rsp, 112), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, WC_L64(rsp, 120), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, rbp, rbp, (unsigned long long*)&rbp); + rbp = (word64)((rbp << 1) | (r11 >> 63)); + rbp = (word64)(rbp * -19); + r11 = (word64)(r11 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rsp, 128) = (word64)(rcx); + WC_S64(rsp, 136) = (word64)(r9); + WC_S64(rsp, 144) = (word64)(r10); + WC_S64(rsp, 152) = (word64)(r11); + /* Multiply by 121666 */ + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r10 = (word64)(0); + rcx = (word64)(rax); + r9 = (word64)(rdx); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + rax = (word64)(0x1db42); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + r12 = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, WC_L64(rsp, 96), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 104), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 112), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, WC_L64(rsp, 120), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + r13 = (word64)((r13 << 1) | (r11 >> 63)); + r11 = (word64)(r11 & r12); + rax = (word64)(0x13); + WC_X64I_MUL128(rax, rdx, rax, r13); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rsp, 96) = (word64)(rcx); + WC_S64(rsp, 104) = (word64)(r9); + WC_S64(rsp, 112) = (word64)(r10); + WC_S64(rsp, 120) = (word64)(r11); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + rcx = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 128)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 136)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 144)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsp, 152)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + /* Store */ + WC_S64(rsp, 0) = (word64)(rcx); + WC_S64(rsp, 8) = (word64)(r9); + WC_S64(rsp, 16) = (word64)(r10); + WC_S64(rsp, 24) = (word64)(r11); + WC_S64(rsp, 160) = (word64)(WC_L64(rsp, 160) - 1); + if ((sword64)(WC_S64(rsp, 160)) >= (sword64)(0)) { + goto L_curve25519_x64_3; + } + /* Invert */ + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 96); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(4); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(9); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 128); + rsi = (word64)(rsp + 96); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 128); + rsi = (word64)(rsp + 128); + rdx = (word64)(0x13); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 128); + rdx = (word64)(rsp + 96); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(9); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(0x31); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 128); + rsi = (word64)(rsp + 96); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 128); + rsi = (word64)(rsp + 128); + rdx = (word64)(0x63); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 128); + rdx = (word64)(rsp + 96); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(0x31); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(4); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(WC_L64(rsp, 168)); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rsp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + rcx = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rsp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rsp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rsp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rsp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rsp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rsp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rsp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rsp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rsp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rsp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rsp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rsp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rsp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rsp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rsp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r15); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbp = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r11 >> 63)); + rdx = (word64)(rdx * 19); + r11 = (word64)(r11 & rbp); + rbp = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, rcx, rbp, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + rbp = (word64)(0x7fffffffffffffff); + rax = (word64)(r11); + rax = (word64)((word64)((sword64)rax >> 63)); + rax = (word64)(rax & 0x13); + r11 = (word64)(r11 & rbp); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + rax = (word64)(0x7fffffffffffffff); + rdx = (word64)(rcx); + cf = _addcarry_u64(0, rdx, 0x13, (unsigned long long*)&rdx); + rdx = (word64)(r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rdx = (word64)(r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rdx = (word64)(r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rdx = (word64)((word64)((sword64)rdx >> 63)); + rdx = (word64)(rdx & 0x13); + r11 = (word64)(r11 & rax); + cf = _addcarry_u64(0, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + r11 = (word64)(r11 & rax); + /* Store */ + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + rax = (word64)(0); + return (int)(word32)rax; +} + +WOLFSSL_LOCAL void fe_pow22523_x64(fe r, const fe a) +{ + word64 rdi, rsi, rsp, rdx; + XALIGNED(32) WC_X64I_SLOT stk[16]; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 112); + /* pow22523 */ + WC_S64(rsp, 96) = (word64)(rdi); + WC_S64(rsp, 104) = (word64)(rsi); + rdi = (word64)(rsp); + rsi = (word64)(WC_L64(rsp, 104)); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(WC_L64(rsp, 104)); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(4); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(9); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(0x13); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(9); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(0x31); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(0x63); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(0x31); + (void)fe_sq_n_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + (void)fe_sq_x64((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(WC_L64(rsp, 96)); + rsi = (word64)(rsp); + rdx = (word64)(WC_L64(rsp, 104)); + (void)fe_mul_x64((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rsi = (word64)(WC_L64(rsp, 104)); + rdi = (word64)(WC_L64(rsp, 96)); +} + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_p1p1_to_p2_x64(ge_p2* r, const ge_p1p1* p) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ + word64 rdi, rsi, rsp, rcx, rax, rdx = 0, r9, r10, r11, r12, r13, r14, r15, + rbx, r8; + XALIGNED(32) WC_X64I_SLOT stk[4]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + WC_S64(rsp, 0) = (word64)(rdi); + WC_S64(rsp, 8) = (word64)(rsi); + rcx = (word64)(rsi); + rcx = (word64)(rcx + 0x60); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + rsi = (word64)(rsi + 0x40); + rdi = (word64)(rdi + 0x40); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + rcx = (word64)(rsi); + rcx = (word64)(rcx - 0x20); + rdi = (word64)(rdi - 0x20); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_p1p1_to_p3_x64(ge_p3* r, const ge_p1p1* p) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ + word64 rdi, rsi, rsp, rcx, rax, rdx = 0, r9, r10, r11, r12, r13, r14, r15, + rbx, r8; + XALIGNED(32) WC_X64I_SLOT stk[4]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + WC_S64(rsp, 0) = (word64)(rdi); + WC_S64(rsp, 8) = (word64)(rsi); + rcx = (word64)(rsi); + rcx = (word64)(rcx + 0x60); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + rcx = (word64)(rsi); + rcx = (word64)(rcx + 0x20); + rdi = (word64)(rdi + 0x60); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + rsi = (word64)(rsi + 0x40); + rdi = (word64)(rdi - 0x40); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + rcx = (word64)(rsi); + rcx = (word64)(rcx + 0x20); + rdi = (word64)(rdi + 0x20); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_p2_dbl_x64(ge_p1p1* r, const ge_p2* p) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ + word64 rdi, rsi, rsp, rax, rdx = 0, r10, r11, r12, r13, r14, r15, rbx, r9, + r8, rcx; + XALIGNED(32) WC_X64I_SLOT stk[4]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 16); + WC_S64(rsp, 0) = (word64)(rdi); + WC_S64(rsp, 8) = (word64)(rsi); + rdi = (word64)(rdi + 0x40); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* Double */ + rbx = (word64)(0); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, r15, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(rax); + r8 = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r8, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r12, r8, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r14, r8, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + rsi = (word64)(rsi + 0x20); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* Double */ + rbx = (word64)(0); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, r15, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(rax); + r8 = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r8, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r12, r8, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r14, r8, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + rsi = (word64)(rdi); + rdi = (word64)(rdi - 0x20); + /* Add-Sub */ + /* Add */ + r13 = (word64)(r9); + cf = _addcarry_u64(0, r9, WC_L64(rsi, 0), (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, WC_L64(rsi, 8), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsi, 16), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(rsi, 24), (unsigned long long*)&r12); + r8 = (word64)(0); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + r8 = (word64)((r8 << 1) | (r12 >> 63)); + r8 = (word64)(r8 * 0x13); + r12 = (word64)(r12 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* Sub */ + cf = _subborrow_u64(0, r13, WC_L64(rsi, 0), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rsi, 8), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsi, 16), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(rsi, 24), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, r8, r8, (unsigned long long*)&r8); + r8 = (word64)((r8 << 1) | (rbx >> 63)); + r8 = (word64)(r8 * -19); + rbx = (word64)(rbx & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r13, r8, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + WC_S64(rsi, 0) = (word64)(r13); + WC_S64(rsi, 8) = (word64)(r14); + WC_S64(rsi, 16) = (word64)(r15); + WC_S64(rsi, 24) = (word64)(rbx); + rcx = (word64)(WC_L64(rsp, 8)); + rsi = (word64)(rcx); + rsi = (word64)(rsi + 0x20); + rdi = (word64)(rdi - 0x20); + /* Add */ + r9 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)(WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, WC_L64(rcx, 0), (unsigned long long*)&r9); + r11 = (word64)(WC_L64(rsi, 16)); + cf = _addcarry_u64(cf, r10, WC_L64(rcx, 8), (unsigned long long*)&r10); + r12 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(cf, r11, WC_L64(rcx, 16), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, WC_L64(rcx, 24), (unsigned long long*)&r12); + r8 = (word64)(0); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + r8 = (word64)((r8 << 1) | (r12 >> 63)); + r8 = (word64)(r8 * 0x13); + r12 = (word64)(r12 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + /* Square */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* Double */ + rbx = (word64)(0); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, r15, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rdi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(rax); + r8 = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rdi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r8, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rdi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r12, r8, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rdi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r14, r8, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + /* Store */ + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x20); + /* Sub */ + cf = _subborrow_u64(0, r9, WC_L64(rsi, 0), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rsi, 8), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, WC_L64(rsi, 16), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, WC_L64(rsi, 24), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r8, r8, (unsigned long long*)&r8); + r8 = (word64)((r8 << 1) | (r12 >> 63)); + r8 = (word64)(r8 * -19); + r12 = (word64)(r12 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + rcx = (word64)(rcx + 0x40); + /* Square * 2 */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rcx, 8)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rcx, 16)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rcx, 24)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rcx, 16)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rcx, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rcx, 24)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* Double */ + rbx = (word64)(0); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, r15, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(rax); + r8 = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r8, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r12, r8, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r14, r8, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r8 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r12 >> 63)); + rdx = (word64)(rdx * 19); + r12 = (word64)(r12 & r8); + r8 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)(r12); + r12 = (word64)((r12 << 1) | (r11 >> 63)); + r11 = (word64)((r11 << 1) | (r10 >> 63)); + r10 = (word64)((r10 << 1) | (r9 >> 63)); + r9 = (word64)(r9 << 1); + r8 = (word64)(0x7fffffffffffffff); + rax = (word64)(rax >> 62); + r12 = (word64)(r12 & r8); + rax = (word64)(rax * 19); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* Store */ + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x40); + rdi = (word64)(rdi + 0x60); + /* Sub */ + cf = _subborrow_u64(0, r9, WC_L64(rsi, 0), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rsi, 8), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, WC_L64(rsi, 16), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, WC_L64(rsi, 24), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r8, r8, (unsigned long long*)&r8); + r8 = (word64)((r8 << 1) | (r12 >> 63)); + r8 = (word64)(r8 * -19); + r12 = (word64)(r12 & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r9, r8, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_madd_x64(ge_p1p1* r, const ge_p3* p, const ge_precomp* q) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ + word64 rdi, rsi, rcx, rsp, r8, r10, r11, r12, r13, r14, r15, rbx, rbp, r9, + rax, rdx = 0; + XALIGNED(32) WC_X64I_SLOT stk[4]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + rcx = (word64)(size_t)q; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 24); + WC_S64(rsp, 0) = (word64)(rdi); + WC_S64(rsp, 8) = (word64)(rsi); + WC_S64(rsp, 16) = (word64)(rcx); + r8 = (word64)(rsi); + rcx = (word64)(rsi); + rcx = (word64)(rcx + 0x20); + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x20); + /* Add-Sub */ + /* Add */ + r10 = (word64)(WC_L64(rcx, 0)); + r11 = (word64)(WC_L64(rcx, 8)); + r12 = (word64)(WC_L64(rcx, 16)); + r13 = (word64)(WC_L64(rcx, 24)); + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(r8, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(r8, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(r8, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(r8, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(r8, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(r8, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(r8, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(r8, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rsi, 0) = (word64)(r14); + WC_S64(rsi, 8) = (word64)(r15); + WC_S64(rsi, 16) = (word64)(rbx); + WC_S64(rsi, 24) = (word64)(rbp); + rcx = (word64)(WC_L64(rsp, 16)); + rcx = (word64)(rcx + 0x20); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + WC_S64(rsi, 0) = (word64)(r10); + WC_S64(rsi, 8) = (word64)(r11); + WC_S64(rsi, 16) = (word64)(r12); + WC_S64(rsi, 24) = (word64)(r13); + r8 = (word64)(r8 + 0x60); + rcx = (word64)(rcx + 0x20); + rdi = (word64)(rdi + 0x60); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + rcx = (word64)(rcx - 0x40); + rdi = (word64)(rdi - 0x60); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + /* Add-Sub */ + /* Add */ + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(rsi, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsi, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(rsi, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(rsi, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(rsi, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsi, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(rsi, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(rsi, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rsi, 0) = (word64)(r10); + WC_S64(rsi, 8) = (word64)(r11); + WC_S64(rsi, 16) = (word64)(r12); + WC_S64(rsi, 24) = (word64)(r13); + WC_S64(rdi, 0) = (word64)(r14); + WC_S64(rdi, 8) = (word64)(r15); + WC_S64(rdi, 16) = (word64)(rbx); + WC_S64(rdi, 24) = (word64)(rbp); + r8 = (word64)(r8 - 0x20); + /* Double */ + r10 = (word64)(WC_L64(r8, 0)); + r11 = (word64)(WC_L64(r8, 8)); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + r12 = (word64)(WC_L64(r8, 16)); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + r13 = (word64)(WC_L64(r8, 24)); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x60); + rdi = (word64)(rdi + 0x40); + /* Add-Sub */ + /* Add */ + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(rsi, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsi, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(rsi, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(rsi, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(rsi, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsi, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(rsi, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(rsi, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rsi, 0) = (word64)(r14); + WC_S64(rsi, 8) = (word64)(r15); + WC_S64(rsi, 16) = (word64)(rbx); + WC_S64(rsi, 24) = (word64)(rbp); +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_msub_x64(ge_p1p1* r, const ge_p3* p, const ge_precomp* q) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ + word64 rdi, rsi, rcx, rsp, r8, r10, r11, r12, r13, r14, r15, rbx, rbp, r9, + rax, rdx = 0; + XALIGNED(32) WC_X64I_SLOT stk[4]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + rcx = (word64)(size_t)q; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 24); + WC_S64(rsp, 0) = (word64)(rdi); + WC_S64(rsp, 8) = (word64)(rsi); + WC_S64(rsp, 16) = (word64)(rcx); + r8 = (word64)(rsi); + rcx = (word64)(rsi); + rcx = (word64)(rcx + 0x20); + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x20); + /* Add-Sub */ + /* Add */ + r10 = (word64)(WC_L64(rcx, 0)); + r11 = (word64)(WC_L64(rcx, 8)); + r12 = (word64)(WC_L64(rcx, 16)); + r13 = (word64)(WC_L64(rcx, 24)); + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(r8, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(r8, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(r8, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(r8, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(r8, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(r8, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(r8, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(r8, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rsi, 0) = (word64)(r14); + WC_S64(rsi, 8) = (word64)(r15); + WC_S64(rsi, 16) = (word64)(rbx); + WC_S64(rsi, 24) = (word64)(rbp); + rcx = (word64)(WC_L64(rsp, 16)); + rdi = (word64)(rdi + 0x20); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + r8 = (word64)(r8 + 0x60); + rcx = (word64)(rcx + 0x40); + rdi = (word64)(rdi + 0x40); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + rcx = (word64)(rcx - 0x20); + rdi = (word64)(rdi - 0x60); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + /* Add-Sub */ + /* Add */ + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(rsi, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsi, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(rsi, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(rsi, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(rsi, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsi, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(rsi, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(rsi, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rsi, 0) = (word64)(r10); + WC_S64(rsi, 8) = (word64)(r11); + WC_S64(rsi, 16) = (word64)(r12); + WC_S64(rsi, 24) = (word64)(r13); + WC_S64(rdi, 0) = (word64)(r14); + WC_S64(rdi, 8) = (word64)(r15); + WC_S64(rdi, 16) = (word64)(rbx); + WC_S64(rdi, 24) = (word64)(rbp); + r8 = (word64)(r8 - 0x20); + rdi = (word64)(rdi + 0x40); + /* Double */ + r10 = (word64)(WC_L64(r8, 0)); + r11 = (word64)(WC_L64(r8, 8)); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + r12 = (word64)(WC_L64(r8, 16)); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + r13 = (word64)(WC_L64(r8, 24)); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x20); + /* Add-Sub */ + /* Add */ + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(rsi, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsi, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(rsi, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(rsi, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(rsi, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsi, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(rsi, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(rsi, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rsi, 0) = (word64)(r10); + WC_S64(rsi, 8) = (word64)(r11); + WC_S64(rsi, 16) = (word64)(r12); + WC_S64(rsi, 24) = (word64)(r13); + WC_S64(rdi, 0) = (word64)(r14); + WC_S64(rdi, 8) = (word64)(r15); + WC_S64(rdi, 16) = (word64)(rbx); + WC_S64(rdi, 24) = (word64)(rbp); +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_add_x64(ge_p1p1* r, const ge_p3* p, const fe qe_cached) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ + word64 rdi, rsi, rcx, rsp, r8, r10, r11, r12, r13, r14, r15, rbx, rbp, r9, + rax, rdx = 0; + XALIGNED(32) WC_X64I_SLOT stk[4]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + rcx = (word64)(size_t)qe_cached; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 24); + WC_S64(rsp, 0) = (word64)(rdi); + WC_S64(rsp, 8) = (word64)(rsi); + WC_S64(rsp, 16) = (word64)(rcx); + r8 = (word64)(rsi); + rcx = (word64)(rsi); + rcx = (word64)(rcx + 0x20); + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x20); + /* Add-Sub */ + /* Add */ + r10 = (word64)(WC_L64(rcx, 0)); + r11 = (word64)(WC_L64(rcx, 8)); + r12 = (word64)(WC_L64(rcx, 16)); + r13 = (word64)(WC_L64(rcx, 24)); + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(r8, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(r8, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(r8, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(r8, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(r8, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(r8, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(r8, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(r8, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rsi, 0) = (word64)(r14); + WC_S64(rsi, 8) = (word64)(r15); + WC_S64(rsi, 16) = (word64)(rbx); + WC_S64(rsi, 24) = (word64)(rbp); + rcx = (word64)(WC_L64(rsp, 16)); + rcx = (word64)(rcx + 0x20); + rdi = (word64)(rdi + 0x20); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + r8 = (word64)(r8 + 0x60); + rcx = (word64)(rcx + 0x40); + rdi = (word64)(rdi + 0x40); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + rcx = (word64)(rcx - 0x60); + rdi = (word64)(rdi - 0x60); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + /* Add-Sub */ + /* Add */ + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(rsi, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsi, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(rsi, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(rsi, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(rsi, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsi, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(rsi, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(rsi, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rsi, 0) = (word64)(r10); + WC_S64(rsi, 8) = (word64)(r11); + WC_S64(rsi, 16) = (word64)(r12); + WC_S64(rsi, 24) = (word64)(r13); + WC_S64(rdi, 0) = (word64)(r14); + WC_S64(rdi, 8) = (word64)(r15); + WC_S64(rdi, 16) = (word64)(rbx); + WC_S64(rdi, 24) = (word64)(rbp); + r8 = (word64)(r8 - 0x20); + rcx = (word64)(rcx + 0x40); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + rdi = (word64)(rdi + 0x40); + /* Double */ + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x20); + /* Add-Sub */ + /* Add */ + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(rsi, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsi, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(rsi, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(rsi, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(rsi, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsi, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(rsi, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(rsi, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rsi, 0) = (word64)(r14); + WC_S64(rsi, 8) = (word64)(r15); + WC_S64(rsi, 16) = (word64)(rbx); + WC_S64(rsi, 24) = (word64)(rbp); +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_sub_x64(ge_p1p1* r, const ge_p3* p, const fe qe_cached) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ + word64 rdi, rsi, rcx, rsp, r8, r10, r11, r12, r13, r14, r15, rbx, rbp, r9, + rax, rdx = 0; + XALIGNED(32) WC_X64I_SLOT stk[4]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + rcx = (word64)(size_t)qe_cached; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 24); + WC_S64(rsp, 0) = (word64)(rdi); + WC_S64(rsp, 8) = (word64)(rsi); + WC_S64(rsp, 16) = (word64)(rcx); + r8 = (word64)(rsi); + rcx = (word64)(rsi); + rcx = (word64)(rcx + 0x20); + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x20); + /* Add-Sub */ + /* Add */ + r10 = (word64)(WC_L64(rcx, 0)); + r11 = (word64)(WC_L64(rcx, 8)); + r12 = (word64)(WC_L64(rcx, 16)); + r13 = (word64)(WC_L64(rcx, 24)); + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(r8, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(r8, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(r8, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(r8, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(r8, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(r8, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(r8, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(r8, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rsi, 0) = (word64)(r14); + WC_S64(rsi, 8) = (word64)(r15); + WC_S64(rsi, 16) = (word64)(rbx); + WC_S64(rsi, 24) = (word64)(rbp); + rcx = (word64)(WC_L64(rsp, 16)); + rdi = (word64)(rdi + 0x20); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + r8 = (word64)(r8 + 0x60); + rcx = (word64)(rcx + 0x60); + rdi = (word64)(rdi + 0x40); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + rcx = (word64)(rcx - 0x40); + rdi = (word64)(rdi - 0x60); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + /* Add-Sub */ + /* Add */ + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(rsi, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rsi, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(rsi, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(rsi, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(rsi, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rsi, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(rsi, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(rsi, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rsi, 0) = (word64)(r10); + WC_S64(rsi, 8) = (word64)(r11); + WC_S64(rsi, 16) = (word64)(r12); + WC_S64(rsi, 24) = (word64)(r13); + WC_S64(rdi, 0) = (word64)(r14); + WC_S64(rdi, 8) = (word64)(r15); + WC_S64(rdi, 16) = (word64)(rbx); + WC_S64(rdi, 24) = (word64)(rbp); + r8 = (word64)(r8 - 0x20); + rcx = (word64)(rcx + 0x20); + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 0)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 8)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 16)); + rbp = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(r8, 24)); + cf = _addcarry_u64(0, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, rbp); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r9 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r13 >> 63)); + rdx = (word64)(rdx * 19); + r13 = (word64)(r13 & r9); + r9 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + r14 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + WC_X64I_MUL128(rax, rdx, rax, r15); + r15 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* Store */ + /* Double */ + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rsi = (word64)(rdi); + rsi = (word64)(rsi + 0x40); + rdi = (word64)(rdi + 0x60); + /* Add-Sub */ + /* Add */ + r14 = (word64)(r10); + cf = _addcarry_u64(0, r10, WC_L64(rdi, 0), (unsigned long long*)&r10); + r15 = (word64)(r11); + cf = _addcarry_u64(cf, r11, WC_L64(rdi, 8), (unsigned long long*)&r11); + rbx = (word64)(r12); + cf = _addcarry_u64(cf, r12, WC_L64(rdi, 16), (unsigned long long*)&r12); + rbp = (word64)(r13); + cf = _addcarry_u64(cf, r13, WC_L64(rdi, 24), (unsigned long long*)&r13); + r9 = (word64)(0); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (r13 >> 63)); + r9 = (word64)(r9 * 0x13); + r13 = (word64)(r13 & ~(word64)((word64)1 << 63)); + /* Sub modulus (if overflow) */ + cf = _addcarry_u64(0, r10, r9, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* Sub */ + cf = _subborrow_u64(0, r14, WC_L64(rdi, 0), (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, WC_L64(rdi, 8), (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, WC_L64(rdi, 16), (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, WC_L64(rdi, 24), (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)((r9 << 1) | (rbp >> 63)); + r9 = (word64)(r9 * -19); + rbp = (word64)(rbp & ~(word64)((word64)1 << 63)); + /* Add modulus (if underflow) */ + cf = _subborrow_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rsi, 0) = (word64)(r14); + WC_S64(rsi, 8) = (word64)(r15); + WC_S64(rsi, 16) = (word64)(rbx); + WC_S64(rsi, 24) = (word64)(rbp); +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#ifdef HAVE_ED25519 +WOLFSSL_LOCAL void fe_sq2_x64(fe r, fe a) +{ + word64 rdi, rsi, rax, rdx = 0, r8, r9, r10, r11, r12, r13, r14, rcx, r15; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + /* Square * 2 */ + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Double */ + r14 = (word64)(0); + cf = _addcarry_u64(0, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + r15 = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, r15, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r15 = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r15, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r15 = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, r15, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r14); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + r15 = (word64)(0x7fffffffffffffff); + rdx = (word64)((rdx << 1) | (r10 >> 63)); + rdx = (word64)(rdx * 19); + r10 = (word64)(r10 & r15); + r15 = (word64)(rdx); + rax = (word64)(0x26); + WC_X64I_MUL128(rax, rdx, rax, r11); + r11 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + WC_X64I_MUL128(rax, rdx, rax, r12); + r12 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0x26); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + WC_X64I_MUL128(rax, rdx, rax, r13); + r13 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rcx, r15, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + rax = (word64)(r10); + r10 = (word64)((r10 << 1) | (r9 >> 63)); + r9 = (word64)((r9 << 1) | (r8 >> 63)); + r8 = (word64)((r8 << 1) | (rcx >> 63)); + rcx = (word64)(rcx << 1); + r15 = (word64)(0x7fffffffffffffff); + rax = (word64)(rax >> 62); + r10 = (word64)(r10 & r15); + rax = (word64)(rax * 19); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* Store */ + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); +} + +WOLFSSL_LOCAL void sc_reduce_x64(byte* s) +{ + word64 rdi, r8, r9, r10, r11, r12, r13, r14, r15, rcx, rsi, rax, rdx = 0, + rbp, rbx; + unsigned char cf; + + rdi = (word64)(size_t)s; + + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + rcx = (word64)(r15); + rsi = (word64)(0xfffffffffffffff); + rcx = (word64)(rcx >> 56); + r15 = (word64)((r15 << 4) | (r14 >> 60)); + r14 = (word64)((r14 << 4) | (r13 >> 60)); + r13 = (word64)((r13 << 4) | (r12 >> 60)); + r12 = (word64)((r12 << 4) | (r11 >> 60)); + r11 = (word64)(r11 & rsi); + r15 = (word64)(r15 & rsi); + /* Add order times bits 504..511 */ + cf = _subborrow_u64(0, r14, rcx, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(0xeb2106215d086329); + WC_X64I_MUL128(rax, rdx, rax, rcx); + rsi = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + rax = (word64)(0xa7ed9ce5a30a2c13); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Sub product of top 4 words and order */ + rcx = (word64)(0xa7ed9ce5a30a2c13); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rcx); + rbp = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(r13); + WC_X64I_MUL128(rax, rdx, rax, rcx); + rsi = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + rax = (word64)(r14); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + rbx = (word64)(0); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rax = (word64)(r15); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r10, rsi, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + rcx = (word64)(0xeb2106215d086329); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rcx); + rbp = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(r13); + WC_X64I_MUL128(rax, rdx, rax, rcx); + rsi = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + rax = (word64)(r14); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r10, rbp, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + rbp = (word64)(0); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + rax = (word64)(r15); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r11, rsi, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rbx, rax, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + cf = _subborrow_u64(0, r10, r12, (unsigned long long*)&r10); + r12 = (word64)(rbx); + cf = _subborrow_u64(cf, r11, r13, (unsigned long long*)&r11); + r13 = (word64)(rbp); + cf = _subborrow_u64(cf, r12, r14, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r15, (unsigned long long*)&r13); + rcx = (word64)(r13); + rcx = (word64)((word64)((sword64)rcx >> 57)); + /* Conditionally subtract order starting at bit 125 */ + rax = (word64)(0xa000000000000000); + rdx = (word64)(0xcb024c634b9eba7d); + rbx = (word64)(0x29bdf3bd45ef39a); + rbp = (word64)(0x200000000000000); + rax = (word64)(rax & rcx); + rdx = (word64)(rdx & rcx); + rbx = (word64)(rbx & rcx); + rbp = (word64)(rbp & rcx); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rbx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbp, (unsigned long long*)&r13); + /* Move bits 252-376 to own registers */ + rcx = (word64)(0xfffffffffffffff); + r13 = (word64)((r13 << 4) | (r12 >> 60)); + r12 = (word64)((r12 << 4) | (r11 >> 60)); + r11 = (word64)(r11 & rcx); + /* Sub product of top 2 words and order */ + /* * -5812631a5cf5d3ed */ + rcx = (word64)(0xa7ed9ce5a30a2c13); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rcx); + rbx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + rax = (word64)(r13); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* * -14def9dea2f79cd7 */ + rcx = (word64)(0xeb2106215d086329); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rcx); + rbp = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + rax = (word64)(r13); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + /* Add overflows at 2 * 64 */ + rsi = (word64)(0xfffffffffffffff); + r11 = (word64)(r11 & rsi); + cf = _addcarry_u64(0, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rbp, (unsigned long long*)&r11); + /* Subtract top at 2 * 64 */ + cf = _subborrow_u64(0, r10, r12, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, r13, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, rsi, rsi, (unsigned long long*)&rsi); + /* Conditional sub order */ + rax = (word64)(0x5812631a5cf5d3ed); + rdx = (word64)(0x14def9dea2f79cd6); + rbx = (word64)(0x1000000000000000); + rax = (word64)(rax & rsi); + rdx = (word64)(rdx & rsi); + rbx = (word64)(rbx & rsi); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0xfffffffffffffff); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rbx, (unsigned long long*)&r11); + r11 = (word64)(r11 & rax); + /* Store result */ + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); +} + +WOLFSSL_LOCAL void sc_muladd_x64(byte* s, byte* a, byte* b, byte* c) +{ + word64 rdi, rsi, rbp, rcx, rax, rdx = 0, r8, r9, r10, r11, r12, r13, r14, + r15, rbx; + unsigned char cf; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)a; + rbp = (word64)(size_t)b; + rcx = (word64)(size_t)c; + + /* Multiply */ + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rbp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rbp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rbp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rbp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rbp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rbp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rbp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rbp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rbp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rbp, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rbp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rbp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rbp, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rbp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rbp, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rbp, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* Add c to a * b */ + cf = _addcarry_u64(0, r8, WC_L64(rcx, 0), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rcx, 8), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rcx, 16), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, WC_L64(rcx, 24), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rbx = (word64)(r15); + rcx = (word64)(0xfffffffffffffff); + rbx = (word64)(rbx >> 56); + r15 = (word64)((r15 << 4) | (r14 >> 60)); + r14 = (word64)((r14 << 4) | (r13 >> 60)); + r13 = (word64)((r13 << 4) | (r12 >> 60)); + r12 = (word64)((r12 << 4) | (r11 >> 60)); + r11 = (word64)(r11 & rcx); + r15 = (word64)(r15 & rcx); + /* Add order times bits 504..507 */ + cf = _subborrow_u64(0, r14, rbx, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(0xeb2106215d086329); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rcx = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + rax = (word64)(0xa7ed9ce5a30a2c13); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rcx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Sub product of top 4 words and order */ + rbx = (word64)(0xa7ed9ce5a30a2c13); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbp = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(r13); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rcx = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + rax = (word64)(r14); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + rsi = (word64)(0); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + rax = (word64)(r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r10, rcx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + rbx = (word64)(0xeb2106215d086329); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbp = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + rax = (word64)(r13); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rcx = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + rax = (word64)(r14); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r10, rbp, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + rbp = (word64)(0); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + rax = (word64)(r15); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r11, rcx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + cf = _subborrow_u64(0, r10, r12, (unsigned long long*)&r10); + r12 = (word64)(rsi); + cf = _subborrow_u64(cf, r11, r13, (unsigned long long*)&r11); + r13 = (word64)(rbp); + cf = _subborrow_u64(cf, r12, r14, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r15, (unsigned long long*)&r13); + rbx = (word64)(r13); + rbx = (word64)((word64)((sword64)rbx >> 57)); + /* Conditionally subtract order starting at bit 125 */ + rax = (word64)(0xa000000000000000); + rdx = (word64)(0xcb024c634b9eba7d); + rsi = (word64)(0x29bdf3bd45ef39a); + rbp = (word64)(0x200000000000000); + rax = (word64)(rax & rbx); + rdx = (word64)(rdx & rbx); + rsi = (word64)(rsi & rbx); + rbp = (word64)(rbp & rbx); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rsi, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbp, (unsigned long long*)&r13); + /* Move bits 252-376 to own registers */ + rbx = (word64)(0xfffffffffffffff); + r13 = (word64)((r13 << 4) | (r12 >> 60)); + r12 = (word64)((r12 << 4) | (r11 >> 60)); + r11 = (word64)(r11 & rbx); + /* Sub product of top 2 words and order */ + /* * -5812631a5cf5d3ed */ + rbx = (word64)(0xa7ed9ce5a30a2c13); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rsi = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + rax = (word64)(r13); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + /* * -14def9dea2f79cd7 */ + rbx = (word64)(0xeb2106215d086329); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rbx); + rbp = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + rax = (word64)(r13); + WC_X64I_MUL128(rax, rdx, rax, rbx); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rbp, rdx, (unsigned long long*)&rbp); + /* Add overflows at 2 * 64 */ + rcx = (word64)(0xfffffffffffffff); + r11 = (word64)(r11 & rcx); + cf = _addcarry_u64(0, r10, rsi, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rbp, (unsigned long long*)&r11); + /* Subtract top at 2 * 64 */ + cf = _subborrow_u64(0, r10, r12, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, r13, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, rcx, rcx, (unsigned long long*)&rcx); + /* Conditional sub order */ + rax = (word64)(0x5812631a5cf5d3ed); + rdx = (word64)(0x14def9dea2f79cd6); + rsi = (word64)(0x1000000000000000); + rax = (word64)(rax & rcx); + rdx = (word64)(rdx & rcx); + rsi = (word64)(rsi & rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + rax = (word64)(0xfffffffffffffff); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rsi, (unsigned long long*)&r11); + r11 = (word64)(r11 & rax); + /* Store result */ + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); +} + +/* Non-constant time modular inversion. + * + * @param [out] r Resulting number. + * @param [in] a Number to invert. + * @return MP_OKAY on success. + */ +WOLFSSL_LOCAL void fe_invert_nct_x64(word64* r, const word64* a) +{ + word64 rdi, rsi, rsp, rcx, r8, r9, r10, r11, r12, r13, r14, r15, rdx = 0, + rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[68]; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + byte zf7; + byte zf8; + byte zf9; + byte zf10; + byte zf11; + byte zf12; + byte zf13; + byte zf14; + byte zf15; + byte zf16; + byte zf17; + byte zf18; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 544; + rsp = (word64)(rsp - 513); + rcx = (word64)(-19); + r8 = (word64)(-1); + r9 = (word64)(-1); + r10 = (word64)(0x7fffffffffffffff); + r11 = (word64)(WC_L64(rsi, 0)); + r12 = (word64)(WC_L64(rsi, 8)); + r13 = (word64)(WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rsi, 24)); + r15 = (word64)(0); + if ((((byte)r11 & 1)) != (0)) { + goto fe_invert_nct_v_even_end; + } +fe_invert_nct_v_even_start: + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)((r13 >> 1) | (r14 << 63)); + r14 = (word64)(r14 >> 1); + WC_S8(rsp, r15) = (byte)(1); + r15 = (word64)(r15 + 1); + if ((((byte)r11 & 1)) == (0)) { + goto fe_invert_nct_v_even_start; + } +fe_invert_nct_v_even_end: +L_fe_invert_nct_uv_start: + zf1 = r10; + zf2 = r14; + if ((r10) < (r14)) { + goto L_fe_invert_nct_uv_v; + } + if ((zf1) > (zf2)) { + goto L_fe_invert_nct_uv_u; + } + zf3 = r9; + zf4 = r13; + if ((r9) < (r13)) { + goto L_fe_invert_nct_uv_v; + } + if ((zf3) > (zf4)) { + goto L_fe_invert_nct_uv_u; + } + zf5 = r8; + zf6 = r12; + if ((r8) < (r12)) { + goto L_fe_invert_nct_uv_v; + } + if ((zf5) > (zf6)) { + goto L_fe_invert_nct_uv_u; + } + if ((rcx) < (r11)) { + goto L_fe_invert_nct_uv_v; + } +L_fe_invert_nct_uv_u: + WC_S8(rsp, r15) = (byte)(2); + r15 = (word64)(r15 + 1); + cf = _subborrow_u64(0, rcx, r11, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r12, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)(r10 >> 1); + if ((((byte)rcx & 1)) != (0)) { + goto fe_invert_nct_usubv_even_end; + } +fe_invert_nct_usubv_even_start: + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)(r10 >> 1); + WC_S8(rsp, r15) = (byte)(0); + r15 = (word64)(r15 + 1); + if ((((byte)rcx & 1)) == (0)) { + goto fe_invert_nct_usubv_even_start; + } +fe_invert_nct_usubv_even_end: + if ((rcx) != (1)) { + goto L_fe_invert_nct_uv_start; + } + rdx = (word64)(r8); + rdx = (word64)(rdx | r9); + if ((rdx) != (0)) { + goto L_fe_invert_nct_uv_start; + } + rdx = (word64)(rdx | r10); + if ((rdx) != (0)) { + goto L_fe_invert_nct_uv_start; + } + rax = (rax & ~(word64)0xff) | ((word64)(byte)(1) & 0xff); + goto L_fe_invert_nct_uv_end; +L_fe_invert_nct_uv_v: + WC_S8(rsp, r15) = (byte)(3); + r15 = (word64)(r15 + 1); + cf = _subborrow_u64(0, r11, rcx, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, r8, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r9, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, r10, (unsigned long long*)&r14); + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)((r13 >> 1) | (r14 << 63)); + r14 = (word64)(r14 >> 1); + if ((((byte)r11 & 1)) != (0)) { + goto fe_invert_nct_vsubu_even_end; + } +fe_invert_nct_vsubu_even_start: + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)((r13 >> 1) | (r14 << 63)); + r14 = (word64)(r14 >> 1); + WC_S8(rsp, r15) = (byte)(1); + r15 = (word64)(r15 + 1); + if ((((byte)r11 & 1)) == (0)) { + goto fe_invert_nct_vsubu_even_start; + } +fe_invert_nct_vsubu_even_end: + if ((r11) != (1)) { + goto L_fe_invert_nct_uv_start; + } + rdx = (word64)(r12); + rdx = (word64)(rdx | r13); + if ((rdx) != (0)) { + goto L_fe_invert_nct_uv_start; + } + rdx = (word64)(rdx | r14); + if ((rdx) != (0)) { + goto L_fe_invert_nct_uv_start; + } + rax = (rax & ~(word64)0xff) | ((word64)(byte)(0) & 0xff); +L_fe_invert_nct_uv_end: + rcx = (word64)(-19); + r8 = (word64)(-1); + r9 = (word64)(-1); + r10 = (word64)(0x7fffffffffffffff); + r11 = (word64)(1); + r12 = (word64)(0); + r13 = (word64)(0); + r14 = (word64)(0); + WC_S8(rsp, r15) = (byte)(7); + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, 0)) & 0xff); + r15 = (word64)(1); + zf7 = (byte)rdx; + zf8 = 1; + if (((byte)rdx) == (1)) { + goto L_fe_invert_nct_op_div2_d; + } + if ((sword8)(zf7) < (sword8)(zf8)) { + goto L_fe_invert_nct_op_div2_b; + } + zf9 = (byte)rdx; + zf10 = 3; + if (((byte)rdx) == (3)) { + goto L_fe_invert_nct_op_d_sub_b; + } + if ((sword8)(zf9) < (sword8)(zf10)) { + goto L_fe_invert_nct_op_b_sub_d; + } + goto L_fe_invert_nct_op_end; +L_fe_invert_nct_op_b_sub_d: + cf = _subborrow_u64(0, rcx, r11, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r12, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + if ((cf) == 0) { + goto L_fe_invert_nct_op_div2_b; + } + rdx = (word64)(-1); + cf = _addcarry_u64(0, rcx, -19, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + rdx = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); +L_fe_invert_nct_op_div2_b: + if ((((byte)rcx & 1)) == (0)) { + goto L_fe_invert_nct_op_div2_b_mod; + } + cf = _addcarry_u64(0, rcx, -19, (unsigned long long*)&rcx); + rdx = (word64)(-1); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + rdx = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); +L_fe_invert_nct_op_div2_b_mod: + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)(r10 >> 1); + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, r15)) & 0xff); + r15 = (word64)(r15 + 1); + zf11 = (byte)rdx; + zf12 = 1; + if (((byte)rdx) == (1)) { + goto L_fe_invert_nct_op_div2_d; + } + if ((sword8)(zf11) < (sword8)(zf12)) { + goto L_fe_invert_nct_op_div2_b; + } + zf13 = (byte)rdx; + zf14 = 3; + if (((byte)rdx) == (3)) { + goto L_fe_invert_nct_op_d_sub_b; + } + if ((sword8)(zf13) < (sword8)(zf14)) { + goto L_fe_invert_nct_op_b_sub_d; + } + goto L_fe_invert_nct_op_end; +L_fe_invert_nct_op_d_sub_b: + cf = _subborrow_u64(0, r11, rcx, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, r8, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r9, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, r10, (unsigned long long*)&r14); + if ((cf) == 0) { + goto L_fe_invert_nct_op_div2_d; + } + rdx = (word64)(-1); + cf = _addcarry_u64(0, r11, -19, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rdx = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); +L_fe_invert_nct_op_div2_d: + if ((((byte)r11 & 1)) == (0)) { + goto L_fe_invert_nct_op_div2_d_mod; + } + cf = _addcarry_u64(0, r11, -19, (unsigned long long*)&r11); + rdx = (word64)(-1); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rdx = (word64)(0x7fffffffffffffff); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); +L_fe_invert_nct_op_div2_d_mod: + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)((r13 >> 1) | (r14 << 63)); + r14 = (word64)(r14 >> 1); + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, r15)) & 0xff); + r15 = (word64)(r15 + 1); + zf15 = (byte)rdx; + zf16 = 1; + if (((byte)rdx) == (1)) { + goto L_fe_invert_nct_op_div2_d; + } + if ((sword8)(zf15) < (sword8)(zf16)) { + goto L_fe_invert_nct_op_div2_b; + } + zf17 = (byte)rdx; + zf18 = 3; + if (((byte)rdx) == (3)) { + goto L_fe_invert_nct_op_d_sub_b; + } + if ((sword8)(zf17) < (sword8)(zf18)) { + goto L_fe_invert_nct_op_b_sub_d; + } +L_fe_invert_nct_op_end: + if (((byte)rax) != (1)) { + goto L_fe_invert_nct_store_d; + } + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + goto L_fe_invert_nct_store_end; +L_fe_invert_nct_store_d: + WC_S64(rdi, 0) = (word64)(r11); + WC_S64(rdi, 8) = (word64)(r12); + WC_S64(rdi, 16) = (word64)(r13); + WC_S64(rdi, 24) = (word64)(r14); +L_fe_invert_nct_store_end: + ; +} + +#endif /* HAVE_ED25519 */ +#ifdef HAVE_INTEL_AVX2 +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void fe_cmov_table_avx2(fe* r, const fe* base, signed char b) +{ + word64 rdi, rsi, rcx, rbx, rax, rdx, r8, r9, r10, r11, r12, r13, r14, r15; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9; + byte zf1; + byte zf2; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)base; + rcx = (word64)(byte)b; + + rbx = (word64)(0); + rax = (word64)((word64)(sword64)(signed char)(byte)rcx); + rdx = (word64)(word32)((sword32)(word32)rax >> 31); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax ^ ( + byte)rdx) & 0xff); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax - ( + byte)rdx) & 0xff); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)((byte)rax) & 0xff); + y7 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rbx)); + rbx = (word64)(1); + y9 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rbx)); + y3 = y9; + y4 = y9; + y8 = _mm256_setzero_si256(); + y7 = _mm256_permutevar8x32_epi32(y7, y8); + y9 = _mm256_permutevar8x32_epi32(y9, y8); + y0 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y6 = _mm256_cmpeq_epi32(y8, y7); + y5 = _mm256_setzero_si256(); + y3 = _mm256_and_si256(y3, y6); + y4 = _mm256_and_si256(y4, y6); + y8 = y9; + y6 = _mm256_cmpeq_epi32(y8, y7); + y8 = _mm256_add_epi32(y8, y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_or_si256(y3, y0); + y4 = _mm256_or_si256(y4, y1); + y5 = _mm256_or_si256(y5, y2); + y6 = _mm256_cmpeq_epi32(y8, y7); + y8 = _mm256_add_epi32(y8, y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_or_si256(y3, y0); + y4 = _mm256_or_si256(y4, y1); + y5 = _mm256_or_si256(y5, y2); + y6 = _mm256_cmpeq_epi32(y8, y7); + y8 = _mm256_add_epi32(y8, y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_or_si256(y3, y0); + y4 = _mm256_or_si256(y4, y1); + y5 = _mm256_or_si256(y5, y2); + y6 = _mm256_cmpeq_epi32(y8, y7); + y8 = _mm256_add_epi32(y8, y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_or_si256(y3, y0); + y4 = _mm256_or_si256(y4, y1); + y5 = _mm256_or_si256(y5, y2); + y6 = _mm256_cmpeq_epi32(y8, y7); + y8 = _mm256_add_epi32(y8, y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_or_si256(y3, y0); + y4 = _mm256_or_si256(y4, y1); + y5 = _mm256_or_si256(y5, y2); + y6 = _mm256_cmpeq_epi32(y8, y7); + y8 = _mm256_add_epi32(y8, y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 512)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 544)); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_or_si256(y3, y0); + y4 = _mm256_or_si256(y4, y1); + y5 = _mm256_or_si256(y5, y2); + y6 = _mm256_cmpeq_epi32(y8, y7); + y8 = _mm256_add_epi32(y8, y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 576)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 608)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 640)); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_or_si256(y3, y0); + y4 = _mm256_or_si256(y4, y1); + y5 = _mm256_or_si256(y5, y2); + y6 = _mm256_cmpeq_epi32(y8, y7); + y8 = _mm256_add_epi32(y8, y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 672)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 704)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 736)); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_or_si256(y3, y0); + y4 = _mm256_or_si256(y4, y1); + y5 = _mm256_or_si256(y5, y2); + rax = (word64)((word64)(sword64)(signed char)(byte)rcx); + rax = (word64)((word64)((sword64)rax >> 63)); + y6 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + y8 = _mm256_setzero_si256(); + y6 = _mm256_permutevar8x32_epi32(y6, y8); + y8 = _mm256_xor_si256(y3, y4); + y8 = _mm256_and_si256(y8, y6); + y3 = _mm256_xor_si256(y3, y8); + y4 = _mm256_xor_si256(y4, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y5); + r8 = (word64)(WC_L64(rdi, 64)); + r9 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(WC_L64(rdi, 80)); + r11 = (word64)(WC_L64(rdi, 88)); + r12 = (word64)(-19); + r13 = (word64)(-1); + r14 = (word64)(-1); + r15 = (word64)(0x7fffffffffffffff); + cf = _subborrow_u64(0, r12, r8, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r9, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, r10, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, r11, (unsigned long long*)&r15); + zf1 = (byte)rcx; + zf2 = 0; + r8 = (sword8)((byte)rcx) < (sword8)(0) ? r12 : r8; + r9 = (sword8)(zf1) < (sword8)(zf2) ? r13 : r9; + r10 = (sword8)(zf1) < (sword8)(zf2) ? r14 : r10; + r11 = (sword8)(zf1) < (sword8)(zf2) ? r15 : r11; + WC_S64(rdi, 64) = (word64)(r8); + WC_S64(rdi, 72) = (word64)(r9); + WC_S64(rdi, 80) = (word64)(r10); + WC_S64(rdi, 88) = (word64)(r11); +} + +WOLFSSL_LOCAL void fe_mul_avx2(fe r, fe a, fe b) +{ + fe_mul_x64(r, a, b); +} + +WOLFSSL_LOCAL void fe_sq_avx2(fe r, fe a) +{ + fe_sq_x64(r, a); +} + +WOLFSSL_LOCAL void fe_sq_n_avx2(fe r, const fe a, word64 n) +{ + fe_sq_n_x64(r, a, n); +} + +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void fe_mul121666_avx2(fe r, fe a) +{ + word64 rdi, rsi, rdx, rax = 0, r13 = 0, rcx = 0, r12 = 0, r8 = 0, r11 = 0, + r9 = 0, r10 = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(0x1db42); + WC_X64I_MUL128(rax, r13, rdx, WC_L64(rsi, 0)); + WC_X64I_MUL128(rcx, r12, rdx, WC_L64(rsi, 8)); + WC_X64I_MUL128(r8, r11, rdx, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, rcx, r13, (unsigned long long*)&rcx); + WC_X64I_MUL128(r9, r10, rdx, WC_L64(rsi, 24)); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r10 = (word64)((r10 << 1) | (r9 >> 63)); + r9 = (word64)(r9 & ~(word64)((word64)1 << 63)); + r10 = (word64)(r10 * 19); + cf = _addcarry_u64(0, rax, r10, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); +} + +WOLFSSL_LOCAL void fe_invert_avx2(fe r, const fe a) +{ + word64 rdi, rsi, rsp, rdx; + XALIGNED(32) WC_X64I_SLOT stk[20]; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 144); + /* Invert */ + WC_S64(rsp, 128) = (word64)(rdi); + WC_S64(rsp, 136) = (word64)(rsi); + rdi = (word64)(rsp); + rsi = (word64)(WC_L64(rsp, 136)); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(WC_L64(rsp, 136)); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp + 64); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(4); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(9); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(0x13); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(9); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(0x31); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 64); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 96); + rsi = (word64)(rsp + 96); + rdx = (word64)(0x63); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 96); + rdx = (word64)(rsp + 64); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(0x31); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(4); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(WC_L64(rsp, 128)); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rsi = (word64)(WC_L64(rsp, 136)); + rdi = (word64)(WC_L64(rsp, 128)); +} + +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +XALIGNED(32) static const word64 L_curve25519_base_avx2_x2[] WC_X64I_UNUSED = { + 0x5cae469cdd684efbULL, 0x8f3f5ced1e350b5cULL, + 0xd9750c687d157114ULL, 0x20d342d51873f1b7ULL, +}; + +WOLFSSL_LOCAL int curve25519_base_avx2(byte* r, byte* n) +{ + return curve25519_base_x64(r, n); +} + +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ +WOLFSSL_LOCAL int curve25519_avx2(byte* r, byte* n, byte* a) +{ + return curve25519_x64(r, n, a); +} + +WOLFSSL_LOCAL void fe_pow22523_avx2(fe r, const fe a) +{ + word64 rdi, rsi, rsp, rdx; + XALIGNED(32) WC_X64I_SLOT stk[16]; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 112); + /* pow22523 */ + WC_S64(rsp, 96) = (word64)(rdi); + WC_S64(rsp, 104) = (word64)(rsi); + rdi = (word64)(rsp); + rsi = (word64)(WC_L64(rsp, 104)); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(WC_L64(rsp, 104)); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(4); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(9); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(0x13); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(9); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(0x31); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 64); + rsi = (word64)(rsp + 64); + rdx = (word64)(0x63); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 64); + rdx = (word64)(rsp + 32); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp + 32); + rsi = (word64)(rsp + 32); + rdx = (word64)(0x31); + (void)fe_sq_n_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (word64)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp + 32); + rdx = (word64)(rsp); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(rsp); + rsi = (word64)(rsp); + (void)fe_sq_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + rdi = (word64)(WC_L64(rsp, 96)); + rsi = (word64)(rsp); + rdx = (word64)(WC_L64(rsp, 104)); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + rsi = (word64)(WC_L64(rsp, 104)); + rdi = (word64)(WC_L64(rsp, 96)); +} + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_p1p1_to_p2_avx2(ge_p2* r, const ge_p1p1* p) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ge_p1p1_to_p2_x64(r, p); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_p1p1_to_p3_avx2(ge_p3* r, const ge_p1p1* p) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ge_p1p1_to_p3_x64(r, p); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_p2_dbl_avx2(ge_p1p1* r, const ge_p2* p) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ge_p2_dbl_x64(r, p); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_madd_avx2(ge_p1p1* r, const ge_p3* p, const ge_precomp* q) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ge_madd_x64(r, p, q); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_msub_avx2(ge_p1p1* r, const ge_p3* p, const ge_precomp* q) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ge_msub_x64(r, p, q); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_add_avx2(ge_p1p1* r, const ge_p3* p, const fe qe_cached) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ge_add_x64(r, p, qe_cached); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL void ge_sub_avx2(ge_p1p1* r, const ge_p3* p, const fe qe_cached) +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +{ +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) + ge_sub_x64(r, p, qe_cached); +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#ifdef HAVE_ED25519 +WOLFSSL_LOCAL void fe_sq2_avx2(fe r, fe a) +{ + fe_sq2_x64(r, a); +} + +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sc_reduce_avx2(byte* s) +{ + word64 rdi, r8, r9, r10, r11, r12, r13, r14, r15, rax, rcx, rdx, rsi = 0, + rbx = 0, rbp; + unsigned char cf; + + rdi = (word64)(size_t)s; + + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + rax = (word64)(r15); + rcx = (word64)(0xfffffffffffffff); + rax = (word64)(rax >> 56); + r15 = (word64)((r15 << 4) | (r14 >> 60)); + r14 = (word64)((r14 << 4) | (r13 >> 60)); + r13 = (word64)((r13 << 4) | (r12 >> 60)); + r12 = (word64)((r12 << 4) | (r11 >> 60)); + r11 = (word64)(r11 & rcx); + r15 = (word64)(r15 & rcx); + /* Add order times bits 504..511 */ + cf = _subborrow_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + rdx = (word64)(0xeb2106215d086329); + WC_X64I_MUL128(rsi, rcx, rdx, rax); + rdx = (word64)(0xa7ed9ce5a30a2c13); + cf = _addcarry_u64(0, r13, rsi, (unsigned long long*)&r13); + WC_X64I_MUL128(rsi, rbx, rdx, rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r12, rsi, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rcx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Sub product of top 4 words and order */ + rdx = (word64)(0xa7ed9ce5a30a2c13); + WC_X64I_MUL128(rcx, rax, rdx, r12); + cf = _addcarry_u64(0, r8, rcx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + WC_X64I_MUL128(rcx, rax, rdx, r14); + cf = _addcarry_u64(cf, r10, rcx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + rsi = (word64)(0); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + WC_X64I_MUL128(rcx, rax, rdx, r13); + cf = _addcarry_u64(0, r9, rcx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + WC_X64I_MUL128(rcx, rax, rdx, r15); + cf = _addcarry_u64(cf, r11, rcx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rsi, rax, (unsigned long long*)&rsi); + rdx = (word64)(0xeb2106215d086329); + WC_X64I_MUL128(rcx, rax, rdx, r12); + cf = _addcarry_u64(0, r9, rcx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + WC_X64I_MUL128(rcx, rax, rdx, r14); + cf = _addcarry_u64(cf, r11, rcx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rsi, rax, (unsigned long long*)&rsi); + rbx = (word64)(0); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + WC_X64I_MUL128(rcx, rax, rdx, r13); + cf = _addcarry_u64(0, r10, rcx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rax, (unsigned long long*)&r11); + WC_X64I_MUL128(rcx, rax, rdx, r15); + cf = _addcarry_u64(cf, rsi, rcx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rax, (unsigned long long*)&rbx); + cf = _subborrow_u64(0, r10, r12, (unsigned long long*)&r10); + r12 = (word64)(rsi); + cf = _subborrow_u64(cf, r11, r13, (unsigned long long*)&r11); + r13 = (word64)(rbx); + cf = _subborrow_u64(cf, r12, r14, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r15, (unsigned long long*)&r13); + rax = (word64)(r13); + rax = (word64)((word64)((sword64)rax >> 57)); + /* Conditionally subtract order starting at bit 125 */ + rsi = (word64)(0xa000000000000000); + rbx = (word64)(0xcb024c634b9eba7d); + rbp = (word64)(0x29bdf3bd45ef39a); + rcx = (word64)(0x200000000000000); + rsi = (word64)(rsi & rax); + rbx = (word64)(rbx & rax); + rbp = (word64)(rbp & rax); + rcx = (word64)(rcx & rax); + cf = _addcarry_u64(0, r9, rsi, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rbp, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rcx, (unsigned long long*)&r13); + /* Move bits 252-376 to own registers */ + rax = (word64)(0xfffffffffffffff); + r13 = (word64)((r13 << 4) | (r12 >> 60)); + r12 = (word64)((r12 << 4) | (r11 >> 60)); + r11 = (word64)(r11 & rax); + /* Sub product of top 2 words and order */ + /* * -5812631a5cf5d3ed */ + rdx = (word64)(0xa7ed9ce5a30a2c13); + WC_X64I_MUL128(rbp, rax, rdx, r12); + rsi = (word64)(0); + cf = _addcarry_u64(0, r8, rbp, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + WC_X64I_MUL128(rbp, rax, rdx, r13); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rsi, rax, (unsigned long long*)&rsi); + /* * -14def9dea2f79cd7 */ + rdx = (word64)(0xeb2106215d086329); + WC_X64I_MUL128(rbp, rax, rdx, r12); + rbx = (word64)(0); + cf = _addcarry_u64(0, r9, rbp, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + WC_X64I_MUL128(rbp, rax, rdx, r13); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _addcarry_u64(0, r10, rbp, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rbx, rax, (unsigned long long*)&rbx); + /* Add overflows at 2 * 64 */ + rcx = (word64)(0xfffffffffffffff); + r11 = (word64)(r11 & rcx); + cf = _addcarry_u64(0, r10, rsi, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rbx, (unsigned long long*)&r11); + /* Subtract top at 2 * 64 */ + cf = _subborrow_u64(0, r10, r12, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, r13, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, rcx, rcx, (unsigned long long*)&rcx); + /* Conditional sub order */ + rsi = (word64)(0x5812631a5cf5d3ed); + rbx = (word64)(0x14def9dea2f79cd6); + rbp = (word64)(0x1000000000000000); + rsi = (word64)(rsi & rcx); + rbx = (word64)(rbx & rcx); + rbp = (word64)(rbp & rcx); + cf = _addcarry_u64(0, r8, rsi, (unsigned long long*)&r8); + rsi = (word64)(0xfffffffffffffff); + cf = _addcarry_u64(cf, r9, rbx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rbp, (unsigned long long*)&r11); + r11 = (word64)(r11 & rsi); + /* Store result */ + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); +} + +WOLFSSL_LOCAL void sc_muladd_avx2(byte* s, byte* a, byte* b, byte* c) +{ + sc_muladd_x64(s, a, b, c); +} + +XALIGNED(16) static const word32 L_fe_invert_nct_avx2_prime[] WC_X64I_UNUSED = { + 0x03ffffed, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x03ffffff, 0x00000000, 0x00000000, 0x00000000, + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x001fffff, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(32) static const word64 L_fe_invert_nct_avx2_one[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(16) static const word32 L_fe_invert_nct_avx2_all_one[] + WC_X64I_UNUSED = { + 0x00000001, 0x00000001, 0x00000001, 0x00000001, + 0x00000001, 0x00000001, 0x00000001, 0x00000001, +}; + +XALIGNED(16) static const word32 L_fe_invert_nct_avx2_mask01111[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000001, 0x00000001, + 0x00000001, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_fe_invert_nct_avx2_down_one_dword[] + WC_X64I_UNUSED = { + 0x00000001, 0x00000002, 0x00000003, 0x00000004, + 0x00000005, 0x00000006, 0x00000007, 0x00000007, +}; + +XALIGNED(16) static const word32 L_fe_invert_nct_avx2_neg[] WC_X64I_UNUSED = { + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x80000000, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_fe_invert_nct_avx2_up_one_dword[] + WC_X64I_UNUSED = { + 0x00000007, 0x00000000, 0x00000001, 0x00000002, + 0x00000003, 0x00000007, 0x00000007, 0x00000007, +}; + +XALIGNED(16) static const word32 L_fe_invert_nct_avx2_mask26[] + WC_X64I_UNUSED = { + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x03ffffff, 0x00000000, 0x00000000, 0x00000000, +}; + +/* Non-constant time modular inversion. + * + * @param [out] r Resulting number. + * @param [in] a Number to invert. + * @param [in] m Modulus. + * @return MP_OKAY on success. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void fe_invert_nct_avx2(word64* r, const word64* a) +{ + word64 rdi, rsi, rax, rcx, r8, r9, r10, r11, r12, r13, rbx, rdx = 0, + r14 = 0, r15 = 0; + __m256i y0, y1, y2, y3, y4 = _mm256_setzero_si256(), + y5 = _mm256_setzero_si256(), y6, y7, y8, y9, y10, y11, y12, y13, + y14; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rax = (word64)(-19); + rcx = (word64)(-1); + r8 = (word64)(-1); + r9 = (word64)(0x7fffffffffffffff); + r10 = (word64)(WC_L64(rsi, 0)); + r11 = (word64)(WC_L64(rsi, 8)); + r12 = (word64)(WC_L64(rsi, 16)); + r13 = (word64)(WC_L64(rsi, 24)); + rbx = (word64)((word64)(size_t)L_fe_invert_nct_avx2_prime); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + rbx = (word64)((word64)(size_t)L_fe_invert_nct_avx2_one); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_fe_invert_nct_avx2_mask01111); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_fe_invert_nct_avx2_all_one); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_fe_invert_nct_avx2_down_one_dword); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_fe_invert_nct_avx2_neg); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_fe_invert_nct_avx2_up_one_dword); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_fe_invert_nct_avx2_mask26); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y0 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y1 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y2 = y8; + y3 = _mm256_zextsi128_si256(_mm_setzero_si128()); + if ((((byte)r10 & 1)) != (0)) { + goto L_fe_invert_nct_avx2_v_even_end; + } +L_fe_invert_nct_avx2_v_even_start: + r10 = (word64)((r10 >> 1) | (r11 << 63)); + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)(r13 >> 1); + if ((_mm256_testz_si256(y2, y8)) == (1)) { + goto L_fe_invert_nct_avx2_v_even_shr1; + } + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); +L_fe_invert_nct_avx2_v_even_shr1: + y4 = _mm256_and_si256(y2, y9); + y5 = _mm256_and_si256(y3, y10); + y4 = _mm256_permutevar8x32_epi32(y4, y11); + y2 = _mm256_srai_epi32(y2, 1); + y3 = _mm256_srai_epi32(y3, 1); + y5 = _mm256_slli_epi32(y5, 25); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y2 = _mm256_add_epi32(y2, y5); + y3 = _mm256_add_epi32(y3, y4); + if ((((byte)r10 & 1)) == (0)) { + goto L_fe_invert_nct_avx2_v_even_start; + } +L_fe_invert_nct_avx2_v_even_end: +L_fe_invert_nct_avx2_uv_start: + zf1 = r9; + zf2 = r13; + if ((r9) < (r13)) { + goto L_fe_invert_nct_avx2_uv_v; + } + if ((zf1) > (zf2)) { + goto L_fe_invert_nct_avx2_uv_u; + } + zf3 = r8; + zf4 = r12; + if ((r8) < (r12)) { + goto L_fe_invert_nct_avx2_uv_v; + } + if ((zf3) > (zf4)) { + goto L_fe_invert_nct_avx2_uv_u; + } + zf5 = rcx; + zf6 = r11; + if ((rcx) < (r11)) { + goto L_fe_invert_nct_avx2_uv_v; + } + if ((zf5) > (zf6)) { + goto L_fe_invert_nct_avx2_uv_u; + } + if ((rax) < (r10)) { + goto L_fe_invert_nct_avx2_uv_v; + } +L_fe_invert_nct_avx2_uv_u: + cf = _subborrow_u64(0, rax, r10, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r11, (unsigned long long*)&rcx); + y0 = _mm256_sub_epi32(y0, y2); + cf = _subborrow_u64(cf, r8, r12, (unsigned long long*)&r8); + y1 = _mm256_sub_epi32(y1, y3); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + if ((_mm256_testz_si256(y1, y12)) == (1)) { + goto L_fe_invert_nct_avx2_usubv_done_neg; + } + y0 = _mm256_add_epi32(y0, y6); + y1 = _mm256_add_epi32(y1, y7); +L_fe_invert_nct_avx2_usubv_done_neg: +L_fe_invert_nct_avx2_usubv_shr1: + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)(r9 >> 1); + if ((_mm256_testz_si256(y0, y8)) == (1)) { + goto L_fe_invert_nct_avx2_usubv_sub_shr1; + } + y0 = _mm256_add_epi32(y0, y6); + y1 = _mm256_add_epi32(y1, y7); +L_fe_invert_nct_avx2_usubv_sub_shr1: + y4 = _mm256_and_si256(y0, y9); + y5 = _mm256_and_si256(y1, y10); + y4 = _mm256_permutevar8x32_epi32(y4, y11); + y0 = _mm256_srai_epi32(y0, 1); + y1 = _mm256_srai_epi32(y1, 1); + y5 = _mm256_slli_epi32(y5, 25); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_add_epi32(y0, y5); + y1 = _mm256_add_epi32(y1, y4); + if ((((byte)rax & 1)) == (0)) { + goto L_fe_invert_nct_avx2_usubv_shr1; + } + if ((rax) != (1)) { + goto L_fe_invert_nct_avx2_uv_start; + } + rdx = (word64)(rcx); + rdx = (word64)(rdx | r8); + if ((rdx) != (0)) { + goto L_fe_invert_nct_avx2_uv_start; + } + rdx = (word64)(rdx | r9); + if ((rdx) != (0)) { + goto L_fe_invert_nct_avx2_uv_start; + } + rax = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 0)); + r8 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 1)); + r10 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2)); + r12 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 3)); + rcx = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 0)); + r9 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 1)); + r11 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 2)); + r13 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 3)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + r14 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 0)); + r15 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 0)); + goto L_fe_invert_nct_avx2_store_done; +L_fe_invert_nct_avx2_uv_v: + cf = _subborrow_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, rcx, (unsigned long long*)&r11); + y2 = _mm256_sub_epi32(y2, y0); + cf = _subborrow_u64(cf, r12, r8, (unsigned long long*)&r12); + y3 = _mm256_sub_epi32(y3, y1); + cf = _subborrow_u64(cf, r13, r9, (unsigned long long*)&r13); + if ((_mm256_testz_si256(y3, y12)) == (1)) { + goto L_fe_invert_nct_avx2_vsubu_done_neg; + } + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); +L_fe_invert_nct_avx2_vsubu_done_neg: +L_fe_invert_nct_avx2_vsubu_shr1: + r10 = (word64)((r10 >> 1) | (r11 << 63)); + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)(r13 >> 1); + if ((_mm256_testz_si256(y2, y8)) == (1)) { + goto L_fe_invert_nct_avx2_vsubu_sub_shr1; + } + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); +L_fe_invert_nct_avx2_vsubu_sub_shr1: + y4 = _mm256_and_si256(y2, y9); + y5 = _mm256_and_si256(y3, y10); + y4 = _mm256_permutevar8x32_epi32(y4, y11); + y2 = _mm256_srai_epi32(y2, 1); + y3 = _mm256_srai_epi32(y3, 1); + y5 = _mm256_slli_epi32(y5, 25); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y2 = _mm256_add_epi32(y2, y5); + y3 = _mm256_add_epi32(y3, y4); + if ((((byte)r10 & 1)) == (0)) { + goto L_fe_invert_nct_avx2_vsubu_shr1; + } + if ((r10) != (1)) { + goto L_fe_invert_nct_avx2_uv_start; + } + rdx = (word64)(r11); + rdx = (word64)(rdx | r12); + if ((rdx) != (0)) { + goto L_fe_invert_nct_avx2_uv_start; + } + rdx = (word64)(rdx | r13); + if ((rdx) != (0)) { + goto L_fe_invert_nct_avx2_uv_start; + } + rax = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 0)); + r8 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 1)); + r10 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 2)); + r12 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 3)); + rcx = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 0)); + r9 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 1)); + r11 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 2)); + r13 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 3)); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y3 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + r14 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 0)); + r15 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 0)); +L_fe_invert_nct_avx2_store_done: + rdx = (word32)((word32)rax); + rax = (word32)((word32)rax & 0x3ffffff); + rdx = (word32)((word32)((sword32)(word32)rdx >> 26)); + rcx = (word32)((word32)rcx + (word32)rdx); + rdx = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0x3ffffff); + rdx = (word32)((word32)((sword32)(word32)rdx >> 26)); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((word32)r8); + r8 = (word32)((word32)r8 & 0x3ffffff); + rdx = (word32)((word32)((sword32)(word32)rdx >> 26)); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((word32)r9); + r9 = (word32)((word32)r9 & 0x3ffffff); + rdx = (word32)((word32)((sword32)(word32)rdx >> 26)); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((word32)r10); + r10 = (word32)((word32)r10 & 0x3ffffff); + rdx = (word32)((word32)((sword32)(word32)rdx >> 26)); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((word32)r11); + r11 = (word32)((word32)r11 & 0x3ffffff); + rdx = (word32)((word32)((sword32)(word32)rdx >> 26)); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((word32)r12); + r12 = (word32)((word32)r12 & 0x3ffffff); + rdx = (word32)((word32)((sword32)(word32)rdx >> 26)); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((word32)r13); + r13 = (word32)((word32)r13 & 0x3ffffff); + rdx = (word32)((word32)((sword32)(word32)rdx >> 26)); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((word32)r14); + r14 = (word32)((word32)r14 & 0x3ffffff); + rdx = (word32)((word32)((sword32)(word32)rdx >> 26)); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word64)((word64)(sword64)(sword32)(word32)rcx); + r9 = (word64)((word64)(sword64)(sword32)(word32)r9); + r11 = (word64)((word64)(sword64)(sword32)(word32)r11); + r13 = (word64)((word64)(sword64)(sword32)(word32)r13); + r15 = (word64)((word64)(sword64)(sword32)(word32)r15); + rcx = (word64)(rcx << 26); + r9 = (word64)(r9 << 26); + r11 = (word64)(r11 << 26); + r13 = (word64)(r13 << 26); + r15 = (word64)(r15 << 26); + rax = (word64)((word64)(sword64)(sword32)(word32)rax); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + r8 = (word64)((word64)(sword64)(sword32)(word32)r8); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)((word64)(sword64)(sword32)(word32)r10); + cf = _addcarry_u64(cf, r10, r11, (unsigned long long*)&r10); + r12 = (word64)((word64)(sword64)(sword32)(word32)r12); + cf = _addcarry_u64(cf, r12, r13, (unsigned long long*)&r12); + r14 = (word64)((word64)(sword64)(sword32)(word32)r14); + cf = _addcarry_u64(cf, r14, r15, (unsigned long long*)&r14); + if ((sword64)(r14) >= (sword64)(0)) { + goto L_fe_invert_nct_avx2_uv_start_no_add_prime; + } + rcx = (word64)(0xfffffffffffed); + r9 = (word64)(0xfffffffffffff); + r11 = (word64)(0xfffffffffffff); + r13 = (word64)(0xfffffffffffff); + r15 = (word64)(0x7fffffffffff); + rax = (word64)(rax + rcx); + r8 = (word64)(r8 + r9); + r10 = (word64)(r10 + r11); + r12 = (word64)(r12 + r13); + r14 = (word64)(r14 + r15); + rdx = (word64)(0xfffffffffffff); + rcx = (word64)(rax); + rax = (word64)(rax & rdx); + rcx = (word64)((word64)((sword64)rcx >> 52)); + r8 = (word64)(r8 + rcx); + r9 = (word64)(r8); + r8 = (word64)(r8 & rdx); + r9 = (word64)((word64)((sword64)r9 >> 52)); + r10 = (word64)(r10 + r9); + r11 = (word64)(r10); + r10 = (word64)(r10 & rdx); + r11 = (word64)((word64)((sword64)r11 >> 52)); + r12 = (word64)(r12 + r11); + r13 = (word64)(r12); + r12 = (word64)(r12 & rdx); + r13 = (word64)((word64)((sword64)r13 >> 52)); + r14 = (word64)(r14 + r13); +L_fe_invert_nct_avx2_uv_start_no_add_prime: + rcx = (word64)(r8); + r9 = (word64)(r10); + r11 = (word64)(r12); + rcx = (word64)(rcx << 52); + r8 = (word64)((word64)((sword64)r8 >> 12)); + r9 = (word64)(r9 << 40); + r10 = (word64)((word64)((sword64)r10 >> 24)); + r11 = (word64)(r11 << 28); + r12 = (word64)((word64)((sword64)r12 >> 36)); + r14 = (word64)(r14 << 16); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r10, r11, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r12, r14, (unsigned long long*)&r12); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r12); + (void)y13; + (void)y14; +} + +#endif /* HAVE_ED25519 */ +#ifdef HAVE_INTEL_AVX512_IFMA +XALIGNED(32) static const word64 L_x25519_ifma_consts[] WC_X64I_UNUSED = { + 0x0007ffffffffffffULL, 0x000fffffffffffdaULL, + 0x000ffffffffffffeULL, 0x0000000000000013ULL, + 0x000000000001db41ULL, +}; + +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +WC_X64I_TARGET("avx512f,avx512vl,avx512ifma") +WOLFSSL_LOCAL int curve25519_base_avx512_ifma(byte* r, byte* n) +{ + word64 rdi, rsi, rsp, r13, r10, r11, r8, rdx, rcx = 0, rax = 0, r9 = 0, + r12 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(), + y16 = _mm256_setzero_si256(), y17 = _mm256_setzero_si256(), y18, + y19, y20, y21, y22, y23 = _mm256_setzero_si256(), + y24 = _mm256_setzero_si256(), y25 = _mm256_setzero_si256(), + y26 = _mm256_setzero_si256(), y27 = _mm256_setzero_si256(), y28, + y29, y30, y31; + XALIGNED(32) WC_X64I_SLOT stk[92]; + __mmask16 k1, k2, k3, k4, k5, k7, k6; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)n; + + rsp = (word64)(size_t)stk + 736; + rsp = (word64)(rsp - 736); + WC_S64(rsp, 712) = (word64)(rdi); + r13 = (word64)(0x7ffffffffffff); + r10 = (word64)(9); + WC_S64(rsp, 0) = (word64)(r10); + WC_S64(rsp, 8) = (word64)(r10); + WC_S64(rsp, 16) = (word64)(r10); + WC_S64(rsp, 24) = (word64)(r10); + WC_S64(rsp, 480) = (word64)(1); + WC_S64(rsp, 488) = (word64)(0); + WC_S64(rsp, 496) = (word64)(r10); + WC_S64(rsp, 504) = (word64)(1); + r10 = (word64)(0); + WC_S64(rsp, 32) = (word64)(r10); + WC_S64(rsp, 40) = (word64)(r10); + WC_S64(rsp, 48) = (word64)(r10); + WC_S64(rsp, 56) = (word64)(r10); + WC_S64(rsp, 512) = (word64)(0); + WC_S64(rsp, 520) = (word64)(0); + WC_S64(rsp, 528) = (word64)(r10); + WC_S64(rsp, 536) = (word64)(0); + WC_S64(rsp, 64) = (word64)(r10); + WC_S64(rsp, 72) = (word64)(r10); + WC_S64(rsp, 80) = (word64)(r10); + WC_S64(rsp, 88) = (word64)(r10); + WC_S64(rsp, 544) = (word64)(0); + WC_S64(rsp, 552) = (word64)(0); + WC_S64(rsp, 560) = (word64)(r10); + WC_S64(rsp, 568) = (word64)(0); + WC_S64(rsp, 96) = (word64)(r10); + WC_S64(rsp, 104) = (word64)(r10); + WC_S64(rsp, 112) = (word64)(r10); + WC_S64(rsp, 120) = (word64)(r10); + WC_S64(rsp, 576) = (word64)(0); + WC_S64(rsp, 584) = (word64)(0); + WC_S64(rsp, 592) = (word64)(r10); + WC_S64(rsp, 600) = (word64)(0); + WC_S64(rsp, 128) = (word64)(r10); + WC_S64(rsp, 136) = (word64)(r10); + WC_S64(rsp, 144) = (word64)(r10); + WC_S64(rsp, 152) = (word64)(r10); + WC_S64(rsp, 608) = (word64)(0); + WC_S64(rsp, 616) = (word64)(0); + WC_S64(rsp, 624) = (word64)(r10); + WC_S64(rsp, 632) = (word64)(0); + r11 = (word64)((word64)(size_t)L_x25519_ifma_consts); + y28 = _mm256_set1_epi64x((long long)WC_L64(r11, 0)); + y29 = _mm256_set1_epi64x((long long)WC_L64(r11, 8)); + y30 = _mm256_set1_epi64x((long long)WC_L64(r11, 16)); + y31 = _mm256_set1_epi64x((long long)WC_L64(r11, 24)); + r10 = (word32)(0xa); + k1 = (__mmask16)(word32)r10; + r10 = (word32)(5); + k2 = (__mmask16)(word32)r10; + r10 = (word32)(4); + k3 = (__mmask16)(word32)r10; + r10 = (word32)(8); + k4 = (__mmask16)(word32)r10; + r10 = (word32)(2); + k5 = (__mmask16)(word32)r10; + r10 = (word32)(6); + k7 = (__mmask16)(word32)r10; + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 512)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 544)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 576)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 608)); + r8 = (word64)(0); + rdx = (word64)(0xfe); +L_curve25519_base_avx512_ifma_bits: + /* Conditionally swap (x2, z2) with (x3, z3) */ + WC_S64(rsp, 704) = (word64)(rdx); + rcx = (word64)(rdx); + rcx = (word64)(rcx & 0x3f); + rdx = (word64)(rdx >> 6); + rax = (word64)(WC_L64(rsi, rdx * 8)); + rax = (word64)(rax >> ((byte)rcx & 63)); + rax = (word64)(rax & 1); + r9 = (word64)(rax); + r8 = (word64)(r8 ^ rax); + r8 = (word64)(0 - r8); + r8 = (word64)(r8 & 0xf); + k6 = (__mmask16)(word32)r8; + r8 = (word64)(r9); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_permute4x64_epi64(y18, 0x4e)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_permute4x64_epi64(y19, 0x4e)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_permute4x64_epi64(y20, 0x4e)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_permute4x64_epi64(y21, 0x4e)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_permute4x64_epi64(y22, 0x4e)); + /* A = x2 + z2, B = x2 - z2, C = x3 + z3, D = x3 - z3 */ + y0 = _mm256_permute4x64_epi64(y18, 0xb1); + y1 = _mm256_permute4x64_epi64(y19, 0xb1); + y2 = _mm256_permute4x64_epi64(y20, 0xb1); + y3 = _mm256_permute4x64_epi64(y21, 0xb1); + y4 = _mm256_permute4x64_epi64(y22, 0xb1); + y23 = _mm256_add_epi64(y0, y18); + y24 = _mm256_add_epi64(y1, y19); + y25 = _mm256_add_epi64(y2, y20); + y26 = _mm256_add_epi64(y3, y21); + y27 = _mm256_add_epi64(y4, y22); + y0 = _mm256_mask_blend_epi64(k1, y0, _mm256_add_epi64(y0, y29)); + y1 = _mm256_mask_blend_epi64(k1, y1, _mm256_add_epi64(y1, y30)); + y2 = _mm256_mask_blend_epi64(k1, y2, _mm256_add_epi64(y2, y30)); + y3 = _mm256_mask_blend_epi64(k1, y3, _mm256_add_epi64(y3, y30)); + y4 = _mm256_mask_blend_epi64(k1, y4, _mm256_add_epi64(y4, y30)); + y23 = _mm256_mask_blend_epi64(k1, y23, _mm256_sub_epi64(y0, y18)); + y24 = _mm256_mask_blend_epi64(k1, y24, _mm256_sub_epi64(y1, y19)); + y25 = _mm256_mask_blend_epi64(k1, y25, _mm256_sub_epi64(y2, y20)); + y26 = _mm256_mask_blend_epi64(k1, y26, _mm256_sub_epi64(y3, y21)); + y27 = _mm256_mask_blend_epi64(k1, y27, _mm256_sub_epi64(y4, y22)); + y5 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y6 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y7 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y8 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y9 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y9, y31); + y24 = _mm256_add_epi64(y24, y5); + y25 = _mm256_add_epi64(y25, y6); + y26 = _mm256_add_epi64(y26, y7); + y27 = _mm256_add_epi64(y27, y8); + /* [AA, BB, CB, DA] = [A, B, C, D] * [A, B, B, A] */ + y18 = _mm256_permute4x64_epi64(y23, 0x14); + y19 = _mm256_permute4x64_epi64(y24, 0x14); + y20 = _mm256_permute4x64_epi64(y25, 0x14); + y21 = _mm256_permute4x64_epi64(y26, 0x14); + y22 = _mm256_permute4x64_epi64(y27, 0x14); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* U = [AA, DA-CB, DA+CB, AA-BB], V = [BB, DA-CB, DA+CB, a24] */ + y23 = _mm256_permute4x64_epi64(y18, 0x69); + y24 = _mm256_permute4x64_epi64(y19, 0x69); + y25 = _mm256_permute4x64_epi64(y20, 0x69); + y26 = _mm256_permute4x64_epi64(y21, 0x69); + y27 = _mm256_permute4x64_epi64(y22, 0x69); + y18 = _mm256_permute4x64_epi64(y18, 0x3c); + y19 = _mm256_permute4x64_epi64(y19, 0x3c); + y20 = _mm256_permute4x64_epi64(y20, 0x3c); + y21 = _mm256_permute4x64_epi64(y21, 0x3c); + y22 = _mm256_permute4x64_epi64(y22, 0x3c); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 160), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 192), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 224), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y22); + y23 = _mm256_mask_blend_epi64(k7, y23, y18); + y24 = _mm256_mask_blend_epi64(k7, y24, y19); + y25 = _mm256_mask_blend_epi64(k7, y25, y20); + y26 = _mm256_mask_blend_epi64(k7, y26, y21); + y27 = _mm256_mask_blend_epi64(k7, y27, y22); + y23 = _mm256_mask_blend_epi64(k4, y23, _mm256_set1_epi64x((long long)WC_L64( + r11, 32))); + y24 = _mm256_mask_blend_epi64(k4, y24, _mm256_setzero_si256()); + y25 = _mm256_mask_blend_epi64(k4, y25, _mm256_setzero_si256()); + y26 = _mm256_mask_blend_epi64(k4, y26, _mm256_setzero_si256()); + y27 = _mm256_mask_blend_epi64(k4, y27, _mm256_setzero_si256()); + /* [AA.BB, GG, FF, a24.E] = U * V */ + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* U3 = [E, E, E, x1], V3 = [aE, AA + a24.E, T, T] */ + y23 = _mm256_permute4x64_epi64(y18, 0x5f); + y24 = _mm256_permute4x64_epi64(y19, 0x5f); + y25 = _mm256_permute4x64_epi64(y20, 0x5f); + y26 = _mm256_permute4x64_epi64(y21, 0x5f); + y27 = _mm256_permute4x64_epi64(y22, 0x5f); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y22); + y5 = _mm256_set1_epi64x((long long)WC_L64(rsp, 160)); + y6 = _mm256_set1_epi64x((long long)WC_L64(rsp, 192)); + y7 = _mm256_set1_epi64x((long long)WC_L64(rsp, 224)); + y8 = _mm256_set1_epi64x((long long)WC_L64(rsp, 256)); + y9 = _mm256_set1_epi64x((long long)WC_L64(rsp, 288)); + y23 = _mm256_mask_blend_epi64(k5, y23, _mm256_add_epi64(y23, y5)); + y24 = _mm256_mask_blend_epi64(k5, y24, _mm256_add_epi64(y24, y6)); + y25 = _mm256_mask_blend_epi64(k5, y25, _mm256_add_epi64(y25, y7)); + y26 = _mm256_mask_blend_epi64(k5, y26, _mm256_add_epi64(y26, y8)); + y27 = _mm256_mask_blend_epi64(k5, y27, _mm256_add_epi64(y27, y9)); + y10 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y11 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y12 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y13 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y14 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y14, y31); + y24 = _mm256_add_epi64(y24, y10); + y25 = _mm256_add_epi64(y25, y11); + y26 = _mm256_add_epi64(y26, y12); + y27 = _mm256_add_epi64(y27, y13); + y18 = _mm256_set1_epi64x((long long)WC_L64(rsp, 184)); + y19 = _mm256_set1_epi64x((long long)WC_L64(rsp, 216)); + y20 = _mm256_set1_epi64x((long long)WC_L64(rsp, 248)); + y21 = _mm256_set1_epi64x((long long)WC_L64(rsp, 280)); + y22 = _mm256_set1_epi64x((long long)WC_L64(rsp, 312)); + y18 = _mm256_mask_blend_epi64(k4, y18, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 0))); + y19 = _mm256_mask_blend_epi64(k4, y19, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 32))); + y20 = _mm256_mask_blend_epi64(k4, y20, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 64))); + y21 = _mm256_mask_blend_epi64(k4, y21, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 96))); + y22 = _mm256_mask_blend_epi64(k4, y22, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 128))); + /* [-, E.H, -, x1.T] = U3 * V3 */ + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* [x2, z2, x3, z3] = [AA.BB, E.H, FF, x1.T] */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y18 = _mm256_mask_blend_epi64(k2, y18, y0); + y19 = _mm256_mask_blend_epi64(k2, y19, y1); + y20 = _mm256_mask_blend_epi64(k2, y20, y2); + y21 = _mm256_mask_blend_epi64(k2, y21, y3); + y22 = _mm256_mask_blend_epi64(k2, y22, y4); + rdx = (word64)(WC_L64(rsp, 704)); + rdx = (word64)(rdx - 1); + if ((sword64)(rdx) >= (sword64)(0)) { + goto L_curve25519_base_avx512_ifma_bits; + } + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 512), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 544), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 576), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 608), y22); + /* Convert to 4 x 64-bit field elements */ + r13 = (word64)(0x7ffffffffffff); + rdx = (word64)(WC_L64(rsp, 480)); + rax = (word64)(WC_L64(rsp, 512)); + r8 = (word64)(WC_L64(rsp, 544)); + r9 = (word64)(WC_L64(rsp, 576)); + r10 = (word64)(WC_L64(rsp, 608)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r13); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r13); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r13); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r13); + r10 = (word64)(r10 + r11); + r11 = (word64)(r10); + r11 = (word64)(r11 >> 51); + r10 = (word64)(r10 & r13); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 640) = (word64)(rdx); + r11 = (word64)(r8); + r11 = (word64)(r11 << 38); + rdx = (word64)(r8); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rsp, 648) = (word64)(r12); + r11 = (word64)(r9); + r11 = (word64)(r11 << 25); + r12 = (word64)(r9); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 656) = (word64)(rdx); + r11 = (word64)(r10); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rsp, 664) = (word64)(r12); + rdx = (word64)(WC_L64(rsp, 488)); + rax = (word64)(WC_L64(rsp, 520)); + r8 = (word64)(WC_L64(rsp, 552)); + r9 = (word64)(WC_L64(rsp, 584)); + r10 = (word64)(WC_L64(rsp, 616)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r13); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r13); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r13); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r13); + r10 = (word64)(r10 + r11); + r11 = (word64)(r10); + r11 = (word64)(r11 >> 51); + r10 = (word64)(r10 & r13); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 672) = (word64)(rdx); + r11 = (word64)(r8); + r11 = (word64)(r11 << 38); + rdx = (word64)(r8); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rsp, 680) = (word64)(r12); + r11 = (word64)(r9); + r11 = (word64)(r11 << 25); + r12 = (word64)(r9); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 688) = (word64)(rdx); + r11 = (word64)(r10); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rsp, 696) = (word64)(r12); + /* z2 = 1 / z2 */ + rdi = (word64)(rsp + 672); + rsi = (word64)(rsp + 672); + (void)fe_invert_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + /* x2 = x2 * z2 */ + rdi = (word64)(rsp + 640); + rsi = (word64)(rsp + 640); + rdx = (word64)(rsp + 672); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + /* Store fully reduced result */ + rdi = (word64)(WC_L64(rsp, 712)); + rsi = (word64)(rsp + 640); + (void)fe_tobytes((unsigned char*)(size_t)rdi, (void*)(size_t)rsi); + rax = (word64)(0); + return (int)(word32)rax; +} + +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ +WC_X64I_TARGET("avx512f,avx512vl,avx512ifma") +WOLFSSL_LOCAL int curve25519_avx512_ifma(byte* r, byte* n, byte* a) +{ + word64 rdi, rsi, r15, rsp, r13, rdx, rax, r8, r9, r10, r12, r11, rcx = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(), + y16 = _mm256_setzero_si256(), y17 = _mm256_setzero_si256(), y18, + y19, y20, y21, y22, y23 = _mm256_setzero_si256(), + y24 = _mm256_setzero_si256(), y25 = _mm256_setzero_si256(), + y26 = _mm256_setzero_si256(), y27 = _mm256_setzero_si256(), y28, + y29, y30, y31; + XALIGNED(32) WC_X64I_SLOT stk[92]; + __mmask16 k1, k2, k3, k4, k5, k7, k6; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)n; + r15 = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 736; + rsp = (word64)(rsp - 736); + WC_S64(rsp, 712) = (word64)(rdi); + r13 = (word64)(0x7ffffffffffff); + rdx = (word64)(WC_L64(r15, 0)); + rax = (word64)(WC_L64(r15, 8)); + r8 = (word64)(WC_L64(r15, 16)); + r9 = (word64)(WC_L64(r15, 24)); + r10 = (word64)(r9); + r10 = (word64)(r10 >> 63); + r10 = (word64)(r10 * 19); + r9 = (word64)(r9 << 1); + r9 = (word64)(r9 >> 1); + r12 = (word64)(rdx); + r12 = (word64)(r12 & r13); + r12 = (word64)(r12 + r10); + WC_S64(rsp, 0) = (word64)(r12); + WC_S64(rsp, 8) = (word64)(r12); + WC_S64(rsp, 16) = (word64)(r12); + WC_S64(rsp, 24) = (word64)(r12); + WC_S64(rsp, 480) = (word64)(1); + WC_S64(rsp, 488) = (word64)(0); + WC_S64(rsp, 496) = (word64)(r12); + WC_S64(rsp, 504) = (word64)(1); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r10 = (word64)(rdx); + r10 = (word64)(r10 & r13); + WC_S64(rsp, 32) = (word64)(r10); + WC_S64(rsp, 40) = (word64)(r10); + WC_S64(rsp, 48) = (word64)(r10); + WC_S64(rsp, 56) = (word64)(r10); + WC_S64(rsp, 512) = (word64)(0); + WC_S64(rsp, 520) = (word64)(0); + WC_S64(rsp, 528) = (word64)(r10); + WC_S64(rsp, 536) = (word64)(0); + rax = (word64)((rax >> 38) | (r8 << 26)); + r10 = (word64)(rax); + r10 = (word64)(r10 & r13); + WC_S64(rsp, 64) = (word64)(r10); + WC_S64(rsp, 72) = (word64)(r10); + WC_S64(rsp, 80) = (word64)(r10); + WC_S64(rsp, 88) = (word64)(r10); + WC_S64(rsp, 544) = (word64)(0); + WC_S64(rsp, 552) = (word64)(0); + WC_S64(rsp, 560) = (word64)(r10); + WC_S64(rsp, 568) = (word64)(0); + r8 = (word64)((r8 >> 25) | (r9 << 39)); + r10 = (word64)(r8); + r10 = (word64)(r10 & r13); + WC_S64(rsp, 96) = (word64)(r10); + WC_S64(rsp, 104) = (word64)(r10); + WC_S64(rsp, 112) = (word64)(r10); + WC_S64(rsp, 120) = (word64)(r10); + WC_S64(rsp, 576) = (word64)(0); + WC_S64(rsp, 584) = (word64)(0); + WC_S64(rsp, 592) = (word64)(r10); + WC_S64(rsp, 600) = (word64)(0); + r9 = (word64)(r9 >> 12); + WC_S64(rsp, 128) = (word64)(r9); + WC_S64(rsp, 136) = (word64)(r9); + WC_S64(rsp, 144) = (word64)(r9); + WC_S64(rsp, 152) = (word64)(r9); + WC_S64(rsp, 608) = (word64)(0); + WC_S64(rsp, 616) = (word64)(0); + WC_S64(rsp, 624) = (word64)(r9); + WC_S64(rsp, 632) = (word64)(0); + r11 = (word64)((word64)(size_t)L_x25519_ifma_consts); + y28 = _mm256_set1_epi64x((long long)WC_L64(r11, 0)); + y29 = _mm256_set1_epi64x((long long)WC_L64(r11, 8)); + y30 = _mm256_set1_epi64x((long long)WC_L64(r11, 16)); + y31 = _mm256_set1_epi64x((long long)WC_L64(r11, 24)); + r10 = (word32)(0xa); + k1 = (__mmask16)(word32)r10; + r10 = (word32)(5); + k2 = (__mmask16)(word32)r10; + r10 = (word32)(4); + k3 = (__mmask16)(word32)r10; + r10 = (word32)(8); + k4 = (__mmask16)(word32)r10; + r10 = (word32)(2); + k5 = (__mmask16)(word32)r10; + r10 = (word32)(6); + k7 = (__mmask16)(word32)r10; + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 512)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 544)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 576)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 608)); + r8 = (word64)(0); + rdx = (word64)(0xfe); +L_curve25519_avx512_ifma_bits: + /* Conditionally swap (x2, z2) with (x3, z3) */ + WC_S64(rsp, 704) = (word64)(rdx); + rcx = (word64)(rdx); + rcx = (word64)(rcx & 0x3f); + rdx = (word64)(rdx >> 6); + rax = (word64)(WC_L64(rsi, rdx * 8)); + rax = (word64)(rax >> ((byte)rcx & 63)); + rax = (word64)(rax & 1); + r9 = (word64)(rax); + r8 = (word64)(r8 ^ rax); + r8 = (word64)(0 - r8); + r8 = (word64)(r8 & 0xf); + k6 = (__mmask16)(word32)r8; + r8 = (word64)(r9); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_permute4x64_epi64(y18, 0x4e)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_permute4x64_epi64(y19, 0x4e)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_permute4x64_epi64(y20, 0x4e)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_permute4x64_epi64(y21, 0x4e)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_permute4x64_epi64(y22, 0x4e)); + /* A = x2 + z2, B = x2 - z2, C = x3 + z3, D = x3 - z3 */ + y0 = _mm256_permute4x64_epi64(y18, 0xb1); + y1 = _mm256_permute4x64_epi64(y19, 0xb1); + y2 = _mm256_permute4x64_epi64(y20, 0xb1); + y3 = _mm256_permute4x64_epi64(y21, 0xb1); + y4 = _mm256_permute4x64_epi64(y22, 0xb1); + y23 = _mm256_add_epi64(y0, y18); + y24 = _mm256_add_epi64(y1, y19); + y25 = _mm256_add_epi64(y2, y20); + y26 = _mm256_add_epi64(y3, y21); + y27 = _mm256_add_epi64(y4, y22); + y0 = _mm256_mask_blend_epi64(k1, y0, _mm256_add_epi64(y0, y29)); + y1 = _mm256_mask_blend_epi64(k1, y1, _mm256_add_epi64(y1, y30)); + y2 = _mm256_mask_blend_epi64(k1, y2, _mm256_add_epi64(y2, y30)); + y3 = _mm256_mask_blend_epi64(k1, y3, _mm256_add_epi64(y3, y30)); + y4 = _mm256_mask_blend_epi64(k1, y4, _mm256_add_epi64(y4, y30)); + y23 = _mm256_mask_blend_epi64(k1, y23, _mm256_sub_epi64(y0, y18)); + y24 = _mm256_mask_blend_epi64(k1, y24, _mm256_sub_epi64(y1, y19)); + y25 = _mm256_mask_blend_epi64(k1, y25, _mm256_sub_epi64(y2, y20)); + y26 = _mm256_mask_blend_epi64(k1, y26, _mm256_sub_epi64(y3, y21)); + y27 = _mm256_mask_blend_epi64(k1, y27, _mm256_sub_epi64(y4, y22)); + y5 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y6 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y7 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y8 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y9 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y9, y31); + y24 = _mm256_add_epi64(y24, y5); + y25 = _mm256_add_epi64(y25, y6); + y26 = _mm256_add_epi64(y26, y7); + y27 = _mm256_add_epi64(y27, y8); + /* [AA, BB, CB, DA] = [A, B, C, D] * [A, B, B, A] */ + y18 = _mm256_permute4x64_epi64(y23, 0x14); + y19 = _mm256_permute4x64_epi64(y24, 0x14); + y20 = _mm256_permute4x64_epi64(y25, 0x14); + y21 = _mm256_permute4x64_epi64(y26, 0x14); + y22 = _mm256_permute4x64_epi64(y27, 0x14); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* U = [AA, DA-CB, DA+CB, AA-BB], V = [BB, DA-CB, DA+CB, a24] */ + y23 = _mm256_permute4x64_epi64(y18, 0x69); + y24 = _mm256_permute4x64_epi64(y19, 0x69); + y25 = _mm256_permute4x64_epi64(y20, 0x69); + y26 = _mm256_permute4x64_epi64(y21, 0x69); + y27 = _mm256_permute4x64_epi64(y22, 0x69); + y18 = _mm256_permute4x64_epi64(y18, 0x3c); + y19 = _mm256_permute4x64_epi64(y19, 0x3c); + y20 = _mm256_permute4x64_epi64(y20, 0x3c); + y21 = _mm256_permute4x64_epi64(y21, 0x3c); + y22 = _mm256_permute4x64_epi64(y22, 0x3c); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 160), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 192), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 224), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y22); + y23 = _mm256_mask_blend_epi64(k7, y23, y18); + y24 = _mm256_mask_blend_epi64(k7, y24, y19); + y25 = _mm256_mask_blend_epi64(k7, y25, y20); + y26 = _mm256_mask_blend_epi64(k7, y26, y21); + y27 = _mm256_mask_blend_epi64(k7, y27, y22); + y23 = _mm256_mask_blend_epi64(k4, y23, _mm256_set1_epi64x((long long)WC_L64( + r11, 32))); + y24 = _mm256_mask_blend_epi64(k4, y24, _mm256_setzero_si256()); + y25 = _mm256_mask_blend_epi64(k4, y25, _mm256_setzero_si256()); + y26 = _mm256_mask_blend_epi64(k4, y26, _mm256_setzero_si256()); + y27 = _mm256_mask_blend_epi64(k4, y27, _mm256_setzero_si256()); + /* [AA.BB, GG, FF, a24.E] = U * V */ + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* U3 = [E, E, E, x1], V3 = [aE, AA + a24.E, T, T] */ + y23 = _mm256_permute4x64_epi64(y18, 0x5f); + y24 = _mm256_permute4x64_epi64(y19, 0x5f); + y25 = _mm256_permute4x64_epi64(y20, 0x5f); + y26 = _mm256_permute4x64_epi64(y21, 0x5f); + y27 = _mm256_permute4x64_epi64(y22, 0x5f); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y22); + y5 = _mm256_set1_epi64x((long long)WC_L64(rsp, 160)); + y6 = _mm256_set1_epi64x((long long)WC_L64(rsp, 192)); + y7 = _mm256_set1_epi64x((long long)WC_L64(rsp, 224)); + y8 = _mm256_set1_epi64x((long long)WC_L64(rsp, 256)); + y9 = _mm256_set1_epi64x((long long)WC_L64(rsp, 288)); + y23 = _mm256_mask_blend_epi64(k5, y23, _mm256_add_epi64(y23, y5)); + y24 = _mm256_mask_blend_epi64(k5, y24, _mm256_add_epi64(y24, y6)); + y25 = _mm256_mask_blend_epi64(k5, y25, _mm256_add_epi64(y25, y7)); + y26 = _mm256_mask_blend_epi64(k5, y26, _mm256_add_epi64(y26, y8)); + y27 = _mm256_mask_blend_epi64(k5, y27, _mm256_add_epi64(y27, y9)); + y10 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y11 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y12 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y13 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y14 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y14, y31); + y24 = _mm256_add_epi64(y24, y10); + y25 = _mm256_add_epi64(y25, y11); + y26 = _mm256_add_epi64(y26, y12); + y27 = _mm256_add_epi64(y27, y13); + y18 = _mm256_set1_epi64x((long long)WC_L64(rsp, 184)); + y19 = _mm256_set1_epi64x((long long)WC_L64(rsp, 216)); + y20 = _mm256_set1_epi64x((long long)WC_L64(rsp, 248)); + y21 = _mm256_set1_epi64x((long long)WC_L64(rsp, 280)); + y22 = _mm256_set1_epi64x((long long)WC_L64(rsp, 312)); + y18 = _mm256_mask_blend_epi64(k4, y18, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 0))); + y19 = _mm256_mask_blend_epi64(k4, y19, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 32))); + y20 = _mm256_mask_blend_epi64(k4, y20, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 64))); + y21 = _mm256_mask_blend_epi64(k4, y21, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 96))); + y22 = _mm256_mask_blend_epi64(k4, y22, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 128))); + /* [-, E.H, -, x1.T] = U3 * V3 */ + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* [x2, z2, x3, z3] = [AA.BB, E.H, FF, x1.T] */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y18 = _mm256_mask_blend_epi64(k2, y18, y0); + y19 = _mm256_mask_blend_epi64(k2, y19, y1); + y20 = _mm256_mask_blend_epi64(k2, y20, y2); + y21 = _mm256_mask_blend_epi64(k2, y21, y3); + y22 = _mm256_mask_blend_epi64(k2, y22, y4); + rdx = (word64)(WC_L64(rsp, 704)); + rdx = (word64)(rdx - 1); + if ((sword64)(rdx) >= (sword64)(0)) { + goto L_curve25519_avx512_ifma_bits; + } + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 512), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 544), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 576), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 608), y22); + /* Convert to 4 x 64-bit field elements */ + r13 = (word64)(0x7ffffffffffff); + rdx = (word64)(WC_L64(rsp, 480)); + rax = (word64)(WC_L64(rsp, 512)); + r8 = (word64)(WC_L64(rsp, 544)); + r9 = (word64)(WC_L64(rsp, 576)); + r10 = (word64)(WC_L64(rsp, 608)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r13); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r13); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r13); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r13); + r10 = (word64)(r10 + r11); + r11 = (word64)(r10); + r11 = (word64)(r11 >> 51); + r10 = (word64)(r10 & r13); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 640) = (word64)(rdx); + r11 = (word64)(r8); + r11 = (word64)(r11 << 38); + rdx = (word64)(r8); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rsp, 648) = (word64)(r12); + r11 = (word64)(r9); + r11 = (word64)(r11 << 25); + r12 = (word64)(r9); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 656) = (word64)(rdx); + r11 = (word64)(r10); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rsp, 664) = (word64)(r12); + rdx = (word64)(WC_L64(rsp, 488)); + rax = (word64)(WC_L64(rsp, 520)); + r8 = (word64)(WC_L64(rsp, 552)); + r9 = (word64)(WC_L64(rsp, 584)); + r10 = (word64)(WC_L64(rsp, 616)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r13); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r13); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r13); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r13); + r10 = (word64)(r10 + r11); + r11 = (word64)(r10); + r11 = (word64)(r11 >> 51); + r10 = (word64)(r10 & r13); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 672) = (word64)(rdx); + r11 = (word64)(r8); + r11 = (word64)(r11 << 38); + rdx = (word64)(r8); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rsp, 680) = (word64)(r12); + r11 = (word64)(r9); + r11 = (word64)(r11 << 25); + r12 = (word64)(r9); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 688) = (word64)(rdx); + r11 = (word64)(r10); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rsp, 696) = (word64)(r12); + /* z2 = 1 / z2 */ + rdi = (word64)(rsp + 672); + rsi = (word64)(rsp + 672); + (void)fe_invert_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + /* x2 = x2 * z2 */ + rdi = (word64)(rsp + 640); + rsi = (word64)(rsp + 640); + rdx = (word64)(rsp + 672); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + /* Store fully reduced result */ + rdi = (word64)(WC_L64(rsp, 712)); + rsi = (word64)(rsp + 640); + (void)fe_tobytes((unsigned char*)(size_t)rdi, (void*)(size_t)rsi); + rax = (word64)(0); + return (int)(word32)rax; +} + +#if defined(WOLFSSL_CURVE25519_NOT_USE_ED25519) +WC_X64I_TARGET("avx512f,avx512vl,avx512ifma,avx512dq") +WOLFSSL_LOCAL int curve25519_base_avx512_ifma_dq(byte* r, byte* n) +{ + word64 rdi, rsi, rsp, r13, r10, r11, r8, rdx, rcx = 0, rax = 0, r9 = 0, + r12 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(), + y16 = _mm256_setzero_si256(), y17 = _mm256_setzero_si256(), y18, + y19, y20, y21, y22, y23 = _mm256_setzero_si256(), + y24 = _mm256_setzero_si256(), y25 = _mm256_setzero_si256(), + y26 = _mm256_setzero_si256(), y27 = _mm256_setzero_si256(), y28, + y29, y30, y31; + XALIGNED(32) WC_X64I_SLOT stk[92]; + __mmask16 k1, k2, k3, k4, k5, k7, k6; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)n; + + rsp = (word64)(size_t)stk + 736; + rsp = (word64)(rsp - 736); + WC_S64(rsp, 712) = (word64)(rdi); + r13 = (word64)(0x7ffffffffffff); + r10 = (word64)(9); + WC_S64(rsp, 0) = (word64)(r10); + WC_S64(rsp, 8) = (word64)(r10); + WC_S64(rsp, 16) = (word64)(r10); + WC_S64(rsp, 24) = (word64)(r10); + WC_S64(rsp, 480) = (word64)(1); + WC_S64(rsp, 488) = (word64)(0); + WC_S64(rsp, 496) = (word64)(r10); + WC_S64(rsp, 504) = (word64)(1); + r10 = (word64)(0); + WC_S64(rsp, 32) = (word64)(r10); + WC_S64(rsp, 40) = (word64)(r10); + WC_S64(rsp, 48) = (word64)(r10); + WC_S64(rsp, 56) = (word64)(r10); + WC_S64(rsp, 512) = (word64)(0); + WC_S64(rsp, 520) = (word64)(0); + WC_S64(rsp, 528) = (word64)(r10); + WC_S64(rsp, 536) = (word64)(0); + WC_S64(rsp, 64) = (word64)(r10); + WC_S64(rsp, 72) = (word64)(r10); + WC_S64(rsp, 80) = (word64)(r10); + WC_S64(rsp, 88) = (word64)(r10); + WC_S64(rsp, 544) = (word64)(0); + WC_S64(rsp, 552) = (word64)(0); + WC_S64(rsp, 560) = (word64)(r10); + WC_S64(rsp, 568) = (word64)(0); + WC_S64(rsp, 96) = (word64)(r10); + WC_S64(rsp, 104) = (word64)(r10); + WC_S64(rsp, 112) = (word64)(r10); + WC_S64(rsp, 120) = (word64)(r10); + WC_S64(rsp, 576) = (word64)(0); + WC_S64(rsp, 584) = (word64)(0); + WC_S64(rsp, 592) = (word64)(r10); + WC_S64(rsp, 600) = (word64)(0); + WC_S64(rsp, 128) = (word64)(r10); + WC_S64(rsp, 136) = (word64)(r10); + WC_S64(rsp, 144) = (word64)(r10); + WC_S64(rsp, 152) = (word64)(r10); + WC_S64(rsp, 608) = (word64)(0); + WC_S64(rsp, 616) = (word64)(0); + WC_S64(rsp, 624) = (word64)(r10); + WC_S64(rsp, 632) = (word64)(0); + r11 = (word64)((word64)(size_t)L_x25519_ifma_consts); + y28 = _mm256_set1_epi64x((long long)WC_L64(r11, 0)); + y29 = _mm256_set1_epi64x((long long)WC_L64(r11, 8)); + y30 = _mm256_set1_epi64x((long long)WC_L64(r11, 16)); + y31 = _mm256_set1_epi64x((long long)WC_L64(r11, 24)); + r10 = (word32)(0xa); + k1 = (__mmask16)(word32)r10; + r10 = (word32)(5); + k2 = (__mmask16)(word32)r10; + r10 = (word32)(4); + k3 = (__mmask16)(word32)r10; + r10 = (word32)(8); + k4 = (__mmask16)(word32)r10; + r10 = (word32)(2); + k5 = (__mmask16)(word32)r10; + r10 = (word32)(6); + k7 = (__mmask16)(word32)r10; + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 512)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 544)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 576)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 608)); + r8 = (word64)(0); + rdx = (word64)(0xfe); +L_curve25519_base_avx512_ifma_dq_bits: + /* Conditionally swap (x2, z2) with (x3, z3) */ + WC_S64(rsp, 704) = (word64)(rdx); + rcx = (word64)(rdx); + rcx = (word64)(rcx & 0x3f); + rdx = (word64)(rdx >> 6); + rax = (word64)(WC_L64(rsi, rdx * 8)); + rax = (word64)(rax >> ((byte)rcx & 63)); + rax = (word64)(rax & 1); + r9 = (word64)(rax); + r8 = (word64)(r8 ^ rax); + r8 = (word64)(0 - r8); + r8 = (word64)(r8 & 0xf); + k6 = (__mmask16)(word32)r8; + r8 = (word64)(r9); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_permute4x64_epi64(y18, 0x4e)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_permute4x64_epi64(y19, 0x4e)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_permute4x64_epi64(y20, 0x4e)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_permute4x64_epi64(y21, 0x4e)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_permute4x64_epi64(y22, 0x4e)); + /* A = x2 + z2, B = x2 - z2, C = x3 + z3, D = x3 - z3 */ + y0 = _mm256_permute4x64_epi64(y18, 0xb1); + y1 = _mm256_permute4x64_epi64(y19, 0xb1); + y2 = _mm256_permute4x64_epi64(y20, 0xb1); + y3 = _mm256_permute4x64_epi64(y21, 0xb1); + y4 = _mm256_permute4x64_epi64(y22, 0xb1); + y23 = _mm256_add_epi64(y0, y18); + y24 = _mm256_add_epi64(y1, y19); + y25 = _mm256_add_epi64(y2, y20); + y26 = _mm256_add_epi64(y3, y21); + y27 = _mm256_add_epi64(y4, y22); + y0 = _mm256_mask_blend_epi64(k1, y0, _mm256_add_epi64(y0, y29)); + y1 = _mm256_mask_blend_epi64(k1, y1, _mm256_add_epi64(y1, y30)); + y2 = _mm256_mask_blend_epi64(k1, y2, _mm256_add_epi64(y2, y30)); + y3 = _mm256_mask_blend_epi64(k1, y3, _mm256_add_epi64(y3, y30)); + y4 = _mm256_mask_blend_epi64(k1, y4, _mm256_add_epi64(y4, y30)); + y23 = _mm256_mask_blend_epi64(k1, y23, _mm256_sub_epi64(y0, y18)); + y24 = _mm256_mask_blend_epi64(k1, y24, _mm256_sub_epi64(y1, y19)); + y25 = _mm256_mask_blend_epi64(k1, y25, _mm256_sub_epi64(y2, y20)); + y26 = _mm256_mask_blend_epi64(k1, y26, _mm256_sub_epi64(y3, y21)); + y27 = _mm256_mask_blend_epi64(k1, y27, _mm256_sub_epi64(y4, y22)); + y5 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y6 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y7 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y8 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y9 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y9, y31); + y24 = _mm256_add_epi64(y24, y5); + y25 = _mm256_add_epi64(y25, y6); + y26 = _mm256_add_epi64(y26, y7); + y27 = _mm256_add_epi64(y27, y8); + /* [AA, BB, CB, DA] = [A, B, C, D] * [A, B, B, A] */ + y18 = _mm256_permute4x64_epi64(y23, 0x14); + y19 = _mm256_permute4x64_epi64(y24, 0x14); + y20 = _mm256_permute4x64_epi64(y25, 0x14); + y21 = _mm256_permute4x64_epi64(y26, 0x14); + y22 = _mm256_permute4x64_epi64(y27, 0x14); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* U = [AA, DA-CB, DA+CB, AA-BB], V = [BB, DA-CB, DA+CB, a24] */ + y23 = _mm256_permute4x64_epi64(y18, 0x69); + y24 = _mm256_permute4x64_epi64(y19, 0x69); + y25 = _mm256_permute4x64_epi64(y20, 0x69); + y26 = _mm256_permute4x64_epi64(y21, 0x69); + y27 = _mm256_permute4x64_epi64(y22, 0x69); + y18 = _mm256_permute4x64_epi64(y18, 0x3c); + y19 = _mm256_permute4x64_epi64(y19, 0x3c); + y20 = _mm256_permute4x64_epi64(y20, 0x3c); + y21 = _mm256_permute4x64_epi64(y21, 0x3c); + y22 = _mm256_permute4x64_epi64(y22, 0x3c); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 160), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 192), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 224), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y22); + y23 = _mm256_mask_blend_epi64(k7, y23, y18); + y24 = _mm256_mask_blend_epi64(k7, y24, y19); + y25 = _mm256_mask_blend_epi64(k7, y25, y20); + y26 = _mm256_mask_blend_epi64(k7, y26, y21); + y27 = _mm256_mask_blend_epi64(k7, y27, y22); + y23 = _mm256_mask_blend_epi64(k4, y23, _mm256_set1_epi64x((long long)WC_L64( + r11, 32))); + y24 = _mm256_mask_blend_epi64(k4, y24, _mm256_setzero_si256()); + y25 = _mm256_mask_blend_epi64(k4, y25, _mm256_setzero_si256()); + y26 = _mm256_mask_blend_epi64(k4, y26, _mm256_setzero_si256()); + y27 = _mm256_mask_blend_epi64(k4, y27, _mm256_setzero_si256()); + /* [AA.BB, GG, FF, a24.E] = U * V */ + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* U3 = [E, E, E, x1], V3 = [aE, AA + a24.E, T, T] */ + y23 = _mm256_permute4x64_epi64(y18, 0x5f); + y24 = _mm256_permute4x64_epi64(y19, 0x5f); + y25 = _mm256_permute4x64_epi64(y20, 0x5f); + y26 = _mm256_permute4x64_epi64(y21, 0x5f); + y27 = _mm256_permute4x64_epi64(y22, 0x5f); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y22); + y5 = _mm256_set1_epi64x((long long)WC_L64(rsp, 160)); + y6 = _mm256_set1_epi64x((long long)WC_L64(rsp, 192)); + y7 = _mm256_set1_epi64x((long long)WC_L64(rsp, 224)); + y8 = _mm256_set1_epi64x((long long)WC_L64(rsp, 256)); + y9 = _mm256_set1_epi64x((long long)WC_L64(rsp, 288)); + y23 = _mm256_mask_blend_epi64(k5, y23, _mm256_add_epi64(y23, y5)); + y24 = _mm256_mask_blend_epi64(k5, y24, _mm256_add_epi64(y24, y6)); + y25 = _mm256_mask_blend_epi64(k5, y25, _mm256_add_epi64(y25, y7)); + y26 = _mm256_mask_blend_epi64(k5, y26, _mm256_add_epi64(y26, y8)); + y27 = _mm256_mask_blend_epi64(k5, y27, _mm256_add_epi64(y27, y9)); + y10 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y11 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y12 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y13 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y14 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y14, y31); + y24 = _mm256_add_epi64(y24, y10); + y25 = _mm256_add_epi64(y25, y11); + y26 = _mm256_add_epi64(y26, y12); + y27 = _mm256_add_epi64(y27, y13); + y18 = _mm256_set1_epi64x((long long)WC_L64(rsp, 184)); + y19 = _mm256_set1_epi64x((long long)WC_L64(rsp, 216)); + y20 = _mm256_set1_epi64x((long long)WC_L64(rsp, 248)); + y21 = _mm256_set1_epi64x((long long)WC_L64(rsp, 280)); + y22 = _mm256_set1_epi64x((long long)WC_L64(rsp, 312)); + y18 = _mm256_mask_blend_epi64(k4, y18, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 0))); + y19 = _mm256_mask_blend_epi64(k4, y19, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 32))); + y20 = _mm256_mask_blend_epi64(k4, y20, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 64))); + y21 = _mm256_mask_blend_epi64(k4, y21, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 96))); + y22 = _mm256_mask_blend_epi64(k4, y22, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 128))); + /* [-, E.H, -, x1.T] = U3 * V3 */ + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* [x2, z2, x3, z3] = [AA.BB, E.H, FF, x1.T] */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y18 = _mm256_mask_blend_epi64(k2, y18, y0); + y19 = _mm256_mask_blend_epi64(k2, y19, y1); + y20 = _mm256_mask_blend_epi64(k2, y20, y2); + y21 = _mm256_mask_blend_epi64(k2, y21, y3); + y22 = _mm256_mask_blend_epi64(k2, y22, y4); + rdx = (word64)(WC_L64(rsp, 704)); + rdx = (word64)(rdx - 1); + if ((sword64)(rdx) >= (sword64)(0)) { + goto L_curve25519_base_avx512_ifma_dq_bits; + } + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 512), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 544), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 576), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 608), y22); + /* Convert to 4 x 64-bit field elements */ + r13 = (word64)(0x7ffffffffffff); + rdx = (word64)(WC_L64(rsp, 480)); + rax = (word64)(WC_L64(rsp, 512)); + r8 = (word64)(WC_L64(rsp, 544)); + r9 = (word64)(WC_L64(rsp, 576)); + r10 = (word64)(WC_L64(rsp, 608)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r13); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r13); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r13); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r13); + r10 = (word64)(r10 + r11); + r11 = (word64)(r10); + r11 = (word64)(r11 >> 51); + r10 = (word64)(r10 & r13); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 640) = (word64)(rdx); + r11 = (word64)(r8); + r11 = (word64)(r11 << 38); + rdx = (word64)(r8); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rsp, 648) = (word64)(r12); + r11 = (word64)(r9); + r11 = (word64)(r11 << 25); + r12 = (word64)(r9); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 656) = (word64)(rdx); + r11 = (word64)(r10); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rsp, 664) = (word64)(r12); + rdx = (word64)(WC_L64(rsp, 488)); + rax = (word64)(WC_L64(rsp, 520)); + r8 = (word64)(WC_L64(rsp, 552)); + r9 = (word64)(WC_L64(rsp, 584)); + r10 = (word64)(WC_L64(rsp, 616)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r13); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r13); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r13); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r13); + r10 = (word64)(r10 + r11); + r11 = (word64)(r10); + r11 = (word64)(r11 >> 51); + r10 = (word64)(r10 & r13); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 672) = (word64)(rdx); + r11 = (word64)(r8); + r11 = (word64)(r11 << 38); + rdx = (word64)(r8); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rsp, 680) = (word64)(r12); + r11 = (word64)(r9); + r11 = (word64)(r11 << 25); + r12 = (word64)(r9); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 688) = (word64)(rdx); + r11 = (word64)(r10); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rsp, 696) = (word64)(r12); + /* z2 = 1 / z2 */ + rdi = (word64)(rsp + 672); + rsi = (word64)(rsp + 672); + (void)fe_invert_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + /* x2 = x2 * z2 */ + rdi = (word64)(rsp + 640); + rsi = (word64)(rsp + 640); + rdx = (word64)(rsp + 672); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + /* Store fully reduced result */ + rdi = (word64)(WC_L64(rsp, 712)); + rsi = (word64)(rsp + 640); + (void)fe_tobytes((unsigned char*)(size_t)rdi, (void*)(size_t)rsi); + rax = (word64)(0); + return (int)(word32)rax; +} + +#endif /* WOLFSSL_CURVE25519_NOT_USE_ED25519 */ +WC_X64I_TARGET("avx512f,avx512vl,avx512ifma,avx512dq") +WOLFSSL_LOCAL int curve25519_avx512_ifma_dq(byte* r, byte* n, byte* a) +{ + word64 rdi, rsi, r15, rsp, r13, rdx, rax, r8, r9, r10, r12, r11, rcx = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(), + y16 = _mm256_setzero_si256(), y17 = _mm256_setzero_si256(), y18, + y19, y20, y21, y22, y23 = _mm256_setzero_si256(), + y24 = _mm256_setzero_si256(), y25 = _mm256_setzero_si256(), + y26 = _mm256_setzero_si256(), y27 = _mm256_setzero_si256(), y28, + y29, y30, y31; + XALIGNED(32) WC_X64I_SLOT stk[92]; + __mmask16 k1, k2, k3, k4, k5, k7, k6; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)n; + r15 = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 736; + rsp = (word64)(rsp - 736); + WC_S64(rsp, 712) = (word64)(rdi); + r13 = (word64)(0x7ffffffffffff); + rdx = (word64)(WC_L64(r15, 0)); + rax = (word64)(WC_L64(r15, 8)); + r8 = (word64)(WC_L64(r15, 16)); + r9 = (word64)(WC_L64(r15, 24)); + r10 = (word64)(r9); + r10 = (word64)(r10 >> 63); + r10 = (word64)(r10 * 19); + r9 = (word64)(r9 << 1); + r9 = (word64)(r9 >> 1); + r12 = (word64)(rdx); + r12 = (word64)(r12 & r13); + r12 = (word64)(r12 + r10); + WC_S64(rsp, 0) = (word64)(r12); + WC_S64(rsp, 8) = (word64)(r12); + WC_S64(rsp, 16) = (word64)(r12); + WC_S64(rsp, 24) = (word64)(r12); + WC_S64(rsp, 480) = (word64)(1); + WC_S64(rsp, 488) = (word64)(0); + WC_S64(rsp, 496) = (word64)(r12); + WC_S64(rsp, 504) = (word64)(1); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r10 = (word64)(rdx); + r10 = (word64)(r10 & r13); + WC_S64(rsp, 32) = (word64)(r10); + WC_S64(rsp, 40) = (word64)(r10); + WC_S64(rsp, 48) = (word64)(r10); + WC_S64(rsp, 56) = (word64)(r10); + WC_S64(rsp, 512) = (word64)(0); + WC_S64(rsp, 520) = (word64)(0); + WC_S64(rsp, 528) = (word64)(r10); + WC_S64(rsp, 536) = (word64)(0); + rax = (word64)((rax >> 38) | (r8 << 26)); + r10 = (word64)(rax); + r10 = (word64)(r10 & r13); + WC_S64(rsp, 64) = (word64)(r10); + WC_S64(rsp, 72) = (word64)(r10); + WC_S64(rsp, 80) = (word64)(r10); + WC_S64(rsp, 88) = (word64)(r10); + WC_S64(rsp, 544) = (word64)(0); + WC_S64(rsp, 552) = (word64)(0); + WC_S64(rsp, 560) = (word64)(r10); + WC_S64(rsp, 568) = (word64)(0); + r8 = (word64)((r8 >> 25) | (r9 << 39)); + r10 = (word64)(r8); + r10 = (word64)(r10 & r13); + WC_S64(rsp, 96) = (word64)(r10); + WC_S64(rsp, 104) = (word64)(r10); + WC_S64(rsp, 112) = (word64)(r10); + WC_S64(rsp, 120) = (word64)(r10); + WC_S64(rsp, 576) = (word64)(0); + WC_S64(rsp, 584) = (word64)(0); + WC_S64(rsp, 592) = (word64)(r10); + WC_S64(rsp, 600) = (word64)(0); + r9 = (word64)(r9 >> 12); + WC_S64(rsp, 128) = (word64)(r9); + WC_S64(rsp, 136) = (word64)(r9); + WC_S64(rsp, 144) = (word64)(r9); + WC_S64(rsp, 152) = (word64)(r9); + WC_S64(rsp, 608) = (word64)(0); + WC_S64(rsp, 616) = (word64)(0); + WC_S64(rsp, 624) = (word64)(r9); + WC_S64(rsp, 632) = (word64)(0); + r11 = (word64)((word64)(size_t)L_x25519_ifma_consts); + y28 = _mm256_set1_epi64x((long long)WC_L64(r11, 0)); + y29 = _mm256_set1_epi64x((long long)WC_L64(r11, 8)); + y30 = _mm256_set1_epi64x((long long)WC_L64(r11, 16)); + y31 = _mm256_set1_epi64x((long long)WC_L64(r11, 24)); + r10 = (word32)(0xa); + k1 = (__mmask16)(word32)r10; + r10 = (word32)(5); + k2 = (__mmask16)(word32)r10; + r10 = (word32)(4); + k3 = (__mmask16)(word32)r10; + r10 = (word32)(8); + k4 = (__mmask16)(word32)r10; + r10 = (word32)(2); + k5 = (__mmask16)(word32)r10; + r10 = (word32)(6); + k7 = (__mmask16)(word32)r10; + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 480)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 512)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 544)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 576)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 608)); + r8 = (word64)(0); + rdx = (word64)(0xfe); +L_curve25519_avx512_ifma_dq_bits: + /* Conditionally swap (x2, z2) with (x3, z3) */ + WC_S64(rsp, 704) = (word64)(rdx); + rcx = (word64)(rdx); + rcx = (word64)(rcx & 0x3f); + rdx = (word64)(rdx >> 6); + rax = (word64)(WC_L64(rsi, rdx * 8)); + rax = (word64)(rax >> ((byte)rcx & 63)); + rax = (word64)(rax & 1); + r9 = (word64)(rax); + r8 = (word64)(r8 ^ rax); + r8 = (word64)(0 - r8); + r8 = (word64)(r8 & 0xf); + k6 = (__mmask16)(word32)r8; + r8 = (word64)(r9); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_permute4x64_epi64(y18, 0x4e)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_permute4x64_epi64(y19, 0x4e)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_permute4x64_epi64(y20, 0x4e)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_permute4x64_epi64(y21, 0x4e)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_permute4x64_epi64(y22, 0x4e)); + /* A = x2 + z2, B = x2 - z2, C = x3 + z3, D = x3 - z3 */ + y0 = _mm256_permute4x64_epi64(y18, 0xb1); + y1 = _mm256_permute4x64_epi64(y19, 0xb1); + y2 = _mm256_permute4x64_epi64(y20, 0xb1); + y3 = _mm256_permute4x64_epi64(y21, 0xb1); + y4 = _mm256_permute4x64_epi64(y22, 0xb1); + y23 = _mm256_add_epi64(y0, y18); + y24 = _mm256_add_epi64(y1, y19); + y25 = _mm256_add_epi64(y2, y20); + y26 = _mm256_add_epi64(y3, y21); + y27 = _mm256_add_epi64(y4, y22); + y0 = _mm256_mask_blend_epi64(k1, y0, _mm256_add_epi64(y0, y29)); + y1 = _mm256_mask_blend_epi64(k1, y1, _mm256_add_epi64(y1, y30)); + y2 = _mm256_mask_blend_epi64(k1, y2, _mm256_add_epi64(y2, y30)); + y3 = _mm256_mask_blend_epi64(k1, y3, _mm256_add_epi64(y3, y30)); + y4 = _mm256_mask_blend_epi64(k1, y4, _mm256_add_epi64(y4, y30)); + y23 = _mm256_mask_blend_epi64(k1, y23, _mm256_sub_epi64(y0, y18)); + y24 = _mm256_mask_blend_epi64(k1, y24, _mm256_sub_epi64(y1, y19)); + y25 = _mm256_mask_blend_epi64(k1, y25, _mm256_sub_epi64(y2, y20)); + y26 = _mm256_mask_blend_epi64(k1, y26, _mm256_sub_epi64(y3, y21)); + y27 = _mm256_mask_blend_epi64(k1, y27, _mm256_sub_epi64(y4, y22)); + y5 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y6 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y7 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y8 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y9 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y9, y31); + y24 = _mm256_add_epi64(y24, y5); + y25 = _mm256_add_epi64(y25, y6); + y26 = _mm256_add_epi64(y26, y7); + y27 = _mm256_add_epi64(y27, y8); + /* [AA, BB, CB, DA] = [A, B, C, D] * [A, B, B, A] */ + y18 = _mm256_permute4x64_epi64(y23, 0x14); + y19 = _mm256_permute4x64_epi64(y24, 0x14); + y20 = _mm256_permute4x64_epi64(y25, 0x14); + y21 = _mm256_permute4x64_epi64(y26, 0x14); + y22 = _mm256_permute4x64_epi64(y27, 0x14); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* U = [AA, DA-CB, DA+CB, AA-BB], V = [BB, DA-CB, DA+CB, a24] */ + y23 = _mm256_permute4x64_epi64(y18, 0x69); + y24 = _mm256_permute4x64_epi64(y19, 0x69); + y25 = _mm256_permute4x64_epi64(y20, 0x69); + y26 = _mm256_permute4x64_epi64(y21, 0x69); + y27 = _mm256_permute4x64_epi64(y22, 0x69); + y18 = _mm256_permute4x64_epi64(y18, 0x3c); + y19 = _mm256_permute4x64_epi64(y19, 0x3c); + y20 = _mm256_permute4x64_epi64(y20, 0x3c); + y21 = _mm256_permute4x64_epi64(y21, 0x3c); + y22 = _mm256_permute4x64_epi64(y22, 0x3c); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 160), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 192), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 224), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y22); + y23 = _mm256_mask_blend_epi64(k7, y23, y18); + y24 = _mm256_mask_blend_epi64(k7, y24, y19); + y25 = _mm256_mask_blend_epi64(k7, y25, y20); + y26 = _mm256_mask_blend_epi64(k7, y26, y21); + y27 = _mm256_mask_blend_epi64(k7, y27, y22); + y23 = _mm256_mask_blend_epi64(k4, y23, _mm256_set1_epi64x((long long)WC_L64( + r11, 32))); + y24 = _mm256_mask_blend_epi64(k4, y24, _mm256_setzero_si256()); + y25 = _mm256_mask_blend_epi64(k4, y25, _mm256_setzero_si256()); + y26 = _mm256_mask_blend_epi64(k4, y26, _mm256_setzero_si256()); + y27 = _mm256_mask_blend_epi64(k4, y27, _mm256_setzero_si256()); + /* [AA.BB, GG, FF, a24.E] = U * V */ + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* U3 = [E, E, E, x1], V3 = [aE, AA + a24.E, T, T] */ + y23 = _mm256_permute4x64_epi64(y18, 0x5f); + y24 = _mm256_permute4x64_epi64(y19, 0x5f); + y25 = _mm256_permute4x64_epi64(y20, 0x5f); + y26 = _mm256_permute4x64_epi64(y21, 0x5f); + y27 = _mm256_permute4x64_epi64(y22, 0x5f); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y22); + y5 = _mm256_set1_epi64x((long long)WC_L64(rsp, 160)); + y6 = _mm256_set1_epi64x((long long)WC_L64(rsp, 192)); + y7 = _mm256_set1_epi64x((long long)WC_L64(rsp, 224)); + y8 = _mm256_set1_epi64x((long long)WC_L64(rsp, 256)); + y9 = _mm256_set1_epi64x((long long)WC_L64(rsp, 288)); + y23 = _mm256_mask_blend_epi64(k5, y23, _mm256_add_epi64(y23, y5)); + y24 = _mm256_mask_blend_epi64(k5, y24, _mm256_add_epi64(y24, y6)); + y25 = _mm256_mask_blend_epi64(k5, y25, _mm256_add_epi64(y25, y7)); + y26 = _mm256_mask_blend_epi64(k5, y26, _mm256_add_epi64(y26, y8)); + y27 = _mm256_mask_blend_epi64(k5, y27, _mm256_add_epi64(y27, y9)); + y10 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y11 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y12 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y13 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y14 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y14, y31); + y24 = _mm256_add_epi64(y24, y10); + y25 = _mm256_add_epi64(y25, y11); + y26 = _mm256_add_epi64(y26, y12); + y27 = _mm256_add_epi64(y27, y13); + y18 = _mm256_set1_epi64x((long long)WC_L64(rsp, 184)); + y19 = _mm256_set1_epi64x((long long)WC_L64(rsp, 216)); + y20 = _mm256_set1_epi64x((long long)WC_L64(rsp, 248)); + y21 = _mm256_set1_epi64x((long long)WC_L64(rsp, 280)); + y22 = _mm256_set1_epi64x((long long)WC_L64(rsp, 312)); + y18 = _mm256_mask_blend_epi64(k4, y18, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 0))); + y19 = _mm256_mask_blend_epi64(k4, y19, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 32))); + y20 = _mm256_mask_blend_epi64(k4, y20, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 64))); + y21 = _mm256_mask_blend_epi64(k4, y21, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 96))); + y22 = _mm256_mask_blend_epi64(k4, y22, _mm256_loadu_si256(( + const __m256i*)WC_PR(rsp, 128))); + /* [-, E.H, -, x1.T] = U3 * V3 */ + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* [x2, z2, x3, z3] = [AA.BB, E.H, FF, x1.T] */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 352)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsp, 448)); + y18 = _mm256_mask_blend_epi64(k2, y18, y0); + y19 = _mm256_mask_blend_epi64(k2, y19, y1); + y20 = _mm256_mask_blend_epi64(k2, y20, y2); + y21 = _mm256_mask_blend_epi64(k2, y21, y3); + y22 = _mm256_mask_blend_epi64(k2, y22, y4); + rdx = (word64)(WC_L64(rsp, 704)); + rdx = (word64)(rdx - 1); + if ((sword64)(rdx) >= (sword64)(0)) { + goto L_curve25519_avx512_ifma_dq_bits; + } + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 512), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 544), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 576), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 608), y22); + /* Convert to 4 x 64-bit field elements */ + r13 = (word64)(0x7ffffffffffff); + rdx = (word64)(WC_L64(rsp, 480)); + rax = (word64)(WC_L64(rsp, 512)); + r8 = (word64)(WC_L64(rsp, 544)); + r9 = (word64)(WC_L64(rsp, 576)); + r10 = (word64)(WC_L64(rsp, 608)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r13); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r13); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r13); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r13); + r10 = (word64)(r10 + r11); + r11 = (word64)(r10); + r11 = (word64)(r11 >> 51); + r10 = (word64)(r10 & r13); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 640) = (word64)(rdx); + r11 = (word64)(r8); + r11 = (word64)(r11 << 38); + rdx = (word64)(r8); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rsp, 648) = (word64)(r12); + r11 = (word64)(r9); + r11 = (word64)(r11 << 25); + r12 = (word64)(r9); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 656) = (word64)(rdx); + r11 = (word64)(r10); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rsp, 664) = (word64)(r12); + rdx = (word64)(WC_L64(rsp, 488)); + rax = (word64)(WC_L64(rsp, 520)); + r8 = (word64)(WC_L64(rsp, 552)); + r9 = (word64)(WC_L64(rsp, 584)); + r10 = (word64)(WC_L64(rsp, 616)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r13); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r13); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r13); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r13); + r10 = (word64)(r10 + r11); + r11 = (word64)(r10); + r11 = (word64)(r11 >> 51); + r10 = (word64)(r10 & r13); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 672) = (word64)(rdx); + r11 = (word64)(r8); + r11 = (word64)(r11 << 38); + rdx = (word64)(r8); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rsp, 680) = (word64)(r12); + r11 = (word64)(r9); + r11 = (word64)(r11 << 25); + r12 = (word64)(r9); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 688) = (word64)(rdx); + r11 = (word64)(r10); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rsp, 696) = (word64)(r12); + /* z2 = 1 / z2 */ + rdi = (word64)(rsp + 672); + rsi = (word64)(rsp + 672); + (void)fe_invert_avx2((void*)(size_t)rdi, (void*)(size_t)rsi); + /* x2 = x2 * z2 */ + rdi = (word64)(rsp + 640); + rsi = (word64)(rsp + 640); + rdx = (word64)(rsp + 672); + (void)fe_mul_avx2((void*)(size_t)rdi, (void*)(size_t)rsi, (void*)( + size_t)rdx); + /* Store fully reduced result */ + rdi = (word64)(WC_L64(rsp, 712)); + rsi = (word64)(rsp + 640); + (void)fe_tobytes((unsigned char*)(size_t)rdi, (void*)(size_t)rsi); + rax = (word64)(0); + return (int)(word32)rax; +} + +#ifdef HAVE_ED25519 +XALIGNED(32) static const word64 L_ge_ifma_consts[] WC_X64I_UNUSED = { + 0x0007ffffffffffffULL, 0x000fffffffffffdaULL, + 0x000ffffffffffffeULL, 0x0000000000000013ULL, + 0x0000000000000001ULL, 0x0000000000000001ULL, + 0x00069b9426b2f159ULL, 0x0000000000000001ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x00035050762add7aULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0003cf44c0038052ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0006738cc7407977ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0002406d9dc56dffULL, 0x0000000000000000ULL, +}; + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WC_X64I_TARGET("avx512f,avx512vl,avx512ifma") +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL int ge_double_scalarmult_vartime_avx512_ifma(ge_p2* r, +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + const byte* a, const ge_p3* A, const byte* b, const ge_precomp* bi, + byte* buf) +{ + word64 rdi, rsi, r14, r15, rbx, rbp, rdx, r11, rax = 0, rcx = 0, r12 = 0, + r8 = 0, r9 = 0, r10 = 0, r13 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(), + y16 = _mm256_setzero_si256(), y17 = _mm256_setzero_si256(), + y18 = _mm256_setzero_si256(), y19 = _mm256_setzero_si256(), + y20 = _mm256_setzero_si256(), y21 = _mm256_setzero_si256(), + y22 = _mm256_setzero_si256(), y23 = _mm256_setzero_si256(), + y24 = _mm256_setzero_si256(), y25 = _mm256_setzero_si256(), + y26 = _mm256_setzero_si256(), y27 = _mm256_setzero_si256(), + y28 = _mm256_setzero_si256(), y29 = _mm256_setzero_si256(), + y30 = _mm256_setzero_si256(), y31 = _mm256_setzero_si256(); + __mmask16 k1, k2, k3, k4, k5, k6, k7; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + r14 = (word64)(size_t)A; + r15 = (word64)(size_t)b; + rbx = (word64)(size_t)bi; + rbp = (word64)(size_t)buf; + + /* Window digits of the scalar multiplying A */ + /* One digit per bit of the scalar */ + rdx = (word64)(0); + r11 = (word64)(0); +L_ge_dsm_a_avx512_ifma_slide_bytes: + rax = (word64)((word64)WC_L8(rsi, r11)); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + r11 = (word64)(r11 + 1); + if ((r11) != (0x20)) { + goto L_ge_dsm_a_avx512_ifma_slide_bytes; + } + /* Fold each run of digits down to one odd digit */ + r11 = (word64)(0); +L_ge_dsm_a_avx512_ifma_slide_digit: + rcx = (word64)((word64)(sword64)(signed char)WC_L8(rbp, r11 + 1920)); + if (((rcx & rcx)) == (0)) { + goto L_ge_dsm_a_avx512_ifma_slide_next_digit; + } + r12 = (word64)(1); +L_ge_dsm_a_avx512_ifma_slide_window: + rdx = (word64)(r11); + rdx = (word64)(rdx + r12); + if ((sword64)(rdx) >= (sword64)(0x100)) { + goto L_ge_dsm_a_avx512_ifma_slide_next_digit; + } + r8 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, rdx + 1920)); + if (((r8 & r8)) == (0)) { + goto L_ge_dsm_a_avx512_ifma_slide_next_window; + } + /* Weight of the digit at i + b, relative to the one at i */ + r9 = (word64)(r8); + r8 = (word64)(r12); +L_ge_dsm_a_avx512_ifma_slide_shift: + r9 = (word64)(r9 + r9); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_ge_dsm_a_avx512_ifma_slide_shift; + } + r9 = (word64)((word64)(sword64)(signed char)(byte)r9); + /* Fold it in if the digit at i stays within range */ + r10 = (word64)(rcx); + r10 = (word64)(r10 + r9); + if ((sword64)(r10) > (sword64)(0xf)) { + goto L_ge_dsm_a_avx512_ifma_slide_sub; + } + rcx = (word64)(r10); + WC_S8(rbp, r11 + 1920) = (byte)((byte)rcx); + WC_S8(rbp, rdx + 1920) = (byte)(0); + goto L_ge_dsm_a_avx512_ifma_slide_next_window; +L_ge_dsm_a_avx512_ifma_slide_sub: + r10 = (word64)(rcx); + r10 = (word64)(r10 - r9); + if ((sword64)(r10) < (sword64)(-15)) { + goto L_ge_dsm_a_avx512_ifma_slide_next_digit; + } + rcx = (word64)(r10); + WC_S8(rbp, r11 + 1920) = (byte)((byte)rcx); + /* Subtracting it borrows from the digits above */ +L_ge_dsm_a_avx512_ifma_slide_carry: + if ((sword64)(rdx) >= (sword64)(0x100)) { + goto L_ge_dsm_a_avx512_ifma_slide_next_window; + } + r10 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, rdx + 1920)); + if (((r10 & r10)) == (0)) { + goto L_ge_dsm_a_avx512_ifma_slide_set; + } + WC_S8(rbp, rdx + 1920) = (byte)(0); + rdx = (word64)(rdx + 1); + goto L_ge_dsm_a_avx512_ifma_slide_carry; +L_ge_dsm_a_avx512_ifma_slide_set: + WC_S8(rbp, rdx + 1920) = (byte)(1); +L_ge_dsm_a_avx512_ifma_slide_next_window: + r12 = (word64)(r12 + 1); + if ((sword64)(r12) <= (sword64)(6)) { + goto L_ge_dsm_a_avx512_ifma_slide_window; + } +L_ge_dsm_a_avx512_ifma_slide_next_digit: + r11 = (word64)(r11 + 1); + if ((sword64)(r11) < (sword64)(0x100)) { + goto L_ge_dsm_a_avx512_ifma_slide_digit; + } + /* Window digits of the base point scalar */ + /* One digit per bit of the scalar */ + rdx = (word64)(0); + r11 = (word64)(0); +L_ge_dsm_b_avx512_ifma_slide_bytes: + rax = (word64)((word64)WC_L8(r15, r11)); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + r11 = (word64)(r11 + 1); + if ((r11) != (0x20)) { + goto L_ge_dsm_b_avx512_ifma_slide_bytes; + } + /* Fold each run of digits down to one odd digit */ + r11 = (word64)(0); +L_ge_dsm_b_avx512_ifma_slide_digit: + rcx = (word64)((word64)(sword64)(signed char)WC_L8(rbp, r11 + 2176)); + if (((rcx & rcx)) == (0)) { + goto L_ge_dsm_b_avx512_ifma_slide_next_digit; + } + r12 = (word64)(1); +L_ge_dsm_b_avx512_ifma_slide_window: + rdx = (word64)(r11); + rdx = (word64)(rdx + r12); + if ((sword64)(rdx) >= (sword64)(0x100)) { + goto L_ge_dsm_b_avx512_ifma_slide_next_digit; + } + r8 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, rdx + 2176)); + if (((r8 & r8)) == (0)) { + goto L_ge_dsm_b_avx512_ifma_slide_next_window; + } + /* Weight of the digit at i + b, relative to the one at i */ + r9 = (word64)(r8); + r8 = (word64)(r12); +L_ge_dsm_b_avx512_ifma_slide_shift: + r9 = (word64)(r9 + r9); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_ge_dsm_b_avx512_ifma_slide_shift; + } + r9 = (word64)((word64)(sword64)(signed char)(byte)r9); + /* Fold it in if the digit at i stays within range */ + r10 = (word64)(rcx); + r10 = (word64)(r10 + r9); + if ((sword64)(r10) > (sword64)(0x3f)) { + goto L_ge_dsm_b_avx512_ifma_slide_sub; + } + rcx = (word64)(r10); + WC_S8(rbp, r11 + 2176) = (byte)((byte)rcx); + WC_S8(rbp, rdx + 2176) = (byte)(0); + goto L_ge_dsm_b_avx512_ifma_slide_next_window; +L_ge_dsm_b_avx512_ifma_slide_sub: + r10 = (word64)(rcx); + r10 = (word64)(r10 - r9); + if ((sword64)(r10) < (sword64)(-63)) { + goto L_ge_dsm_b_avx512_ifma_slide_next_digit; + } + rcx = (word64)(r10); + WC_S8(rbp, r11 + 2176) = (byte)((byte)rcx); + /* Subtracting it borrows from the digits above */ +L_ge_dsm_b_avx512_ifma_slide_carry: + if ((sword64)(rdx) >= (sword64)(0x100)) { + goto L_ge_dsm_b_avx512_ifma_slide_next_window; + } + r10 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, rdx + 2176)); + if (((r10 & r10)) == (0)) { + goto L_ge_dsm_b_avx512_ifma_slide_set; + } + WC_S8(rbp, rdx + 2176) = (byte)(0); + rdx = (word64)(rdx + 1); + goto L_ge_dsm_b_avx512_ifma_slide_carry; +L_ge_dsm_b_avx512_ifma_slide_set: + WC_S8(rbp, rdx + 2176) = (byte)(1); +L_ge_dsm_b_avx512_ifma_slide_next_window: + r12 = (word64)(r12 + 1); + if ((sword64)(r12) <= (sword64)(6)) { + goto L_ge_dsm_b_avx512_ifma_slide_window; + } +L_ge_dsm_b_avx512_ifma_slide_next_digit: + r11 = (word64)(r11 + 1); + if ((sword64)(r11) < (sword64)(0x100)) { + goto L_ge_dsm_b_avx512_ifma_slide_digit; + } + r15 = (word64)((word64)(size_t)L_ge_ifma_consts); + y28 = _mm256_set1_epi64x((long long)WC_L64(r15, 0)); + y29 = _mm256_set1_epi64x((long long)WC_L64(r15, 8)); + y30 = _mm256_set1_epi64x((long long)WC_L64(r15, 16)); + y31 = _mm256_set1_epi64x((long long)WC_L64(r15, 24)); + r9 = (word32)(8); + k1 = (__mmask16)(word32)r9; + r9 = (word32)(0xc); + k2 = (__mmask16)(word32)r9; + r9 = (word32)(9); + k3 = (__mmask16)(word32)r9; + r9 = (word32)(6); + k4 = (__mmask16)(word32)r9; + r9 = (word32)(1); + k5 = (__mmask16)(word32)r9; + r9 = (word32)(2); + k6 = (__mmask16)(word32)r9; + r9 = (word32)(4); + k7 = (__mmask16)(word32)r9; + /* Odd multiples of A, cached and in limb form */ + r10 = (word64)(0x7ffffffffffff); + /* A in limb form: [X, Y, Z, T] */ + rdx = (word64)(WC_L64(r14, 0)); + rax = (word64)(WC_L64(r14, 8)); + rcx = (word64)(WC_L64(r14, 16)); + r8 = (word64)(WC_L64(r14, 24)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1440) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1472) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1504) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1536) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1568) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 32)); + rax = (word64)(WC_L64(r14, 40)); + rcx = (word64)(WC_L64(r14, 48)); + r8 = (word64)(WC_L64(r14, 56)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1448) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1480) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1512) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1544) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1576) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 64)); + rax = (word64)(WC_L64(r14, 72)); + rcx = (word64)(WC_L64(r14, 80)); + r8 = (word64)(WC_L64(r14, 88)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1456) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1488) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1520) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1552) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1584) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 96)); + rax = (word64)(WC_L64(r14, 104)); + rcx = (word64)(WC_L64(r14, 112)); + r8 = (word64)(WC_L64(r14, 120)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1464) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1496) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1528) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1560) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1592) = (word64)(r8); + /* Ai[0] = A */ + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1440)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1472)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1504)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1536)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1568)); + /* To cached */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 32)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 64)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 96)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 128)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 160)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 0), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 32), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 64), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 96), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 128), y22); + /* A2 = 2.A */ + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1440)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1472)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1504)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1536)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1568)); + /* Double */ + y23 = _mm256_permute4x64_epi64(y18, 0x64); + y24 = _mm256_permute4x64_epi64(y19, 0x64); + y25 = _mm256_permute4x64_epi64(y20, 0x64); + y26 = _mm256_permute4x64_epi64(y21, 0x64); + y27 = _mm256_permute4x64_epi64(y22, 0x64); + y18 = _mm256_permute4x64_epi64(y18, 0x24); + y19 = _mm256_permute4x64_epi64(y19, 0x24); + y20 = _mm256_permute4x64_epi64(y20, 0x24); + y21 = _mm256_permute4x64_epi64(y21, 0x24); + y22 = _mm256_permute4x64_epi64(y22, 0x24); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_add_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* Y3 = YY + XX, Z3 = YY - XX */ + y23 = _mm256_permute4x64_epi64(y18, 0x55); + y24 = _mm256_permute4x64_epi64(y19, 0x55); + y25 = _mm256_permute4x64_epi64(y20, 0x55); + y26 = _mm256_permute4x64_epi64(y21, 0x55); + y27 = _mm256_permute4x64_epi64(y22, 0x55); + y0 = _mm256_permute4x64_epi64(y18, 0); + y1 = _mm256_permute4x64_epi64(y19, 0); + y2 = _mm256_permute4x64_epi64(y20, 0); + y3 = _mm256_permute4x64_epi64(y21, 0); + y4 = _mm256_permute4x64_epi64(y22, 0); + y5 = _mm256_add_epi64(y23, y0); + y6 = _mm256_add_epi64(y24, y1); + y7 = _mm256_add_epi64(y25, y2); + y8 = _mm256_add_epi64(y26, y3); + y9 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y30)); + y5 = _mm256_mask_blend_epi64(k2, y5, _mm256_sub_epi64(y23, y0)); + y6 = _mm256_mask_blend_epi64(k2, y6, _mm256_sub_epi64(y24, y1)); + y7 = _mm256_mask_blend_epi64(k2, y7, _mm256_sub_epi64(y25, y2)); + y8 = _mm256_mask_blend_epi64(k2, y8, _mm256_sub_epi64(y26, y3)); + y9 = _mm256_mask_blend_epi64(k2, y9, _mm256_sub_epi64(y27, y4)); + y23 = _mm256_srli_epi64(y5, 51); + y5 = _mm256_and_si256(y5, y28); + y24 = _mm256_srli_epi64(y6, 51); + y6 = _mm256_and_si256(y6, y28); + y25 = _mm256_srli_epi64(y7, 51); + y7 = _mm256_and_si256(y7, y28); + y26 = _mm256_srli_epi64(y8, 51); + y8 = _mm256_and_si256(y8, y28); + y27 = _mm256_srli_epi64(y9, 51); + y9 = _mm256_and_si256(y9, y28); + y5 = _mm256_madd52lo_epu64(y5, y27, y31); + y6 = _mm256_add_epi64(y6, y23); + y7 = _mm256_add_epi64(y7, y24); + y8 = _mm256_add_epi64(y8, y25); + y9 = _mm256_add_epi64(y9, y26); + /* X3 = AA - Y3, T3 = 2.ZZ - Z3 */ + y23 = _mm256_permute4x64_epi64(y18, 0xaf); + y24 = _mm256_permute4x64_epi64(y19, 0xaf); + y25 = _mm256_permute4x64_epi64(y20, 0xaf); + y26 = _mm256_permute4x64_epi64(y21, 0xaf); + y27 = _mm256_permute4x64_epi64(y22, 0xaf); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_sub_epi64(y23, y5)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_sub_epi64(y24, y6)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_sub_epi64(y25, y7)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_sub_epi64(y26, y8)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_sub_epi64(y27, y9)); + y0 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y1 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y2 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y3 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y4 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y4, y31); + y24 = _mm256_add_epi64(y24, y0); + y25 = _mm256_add_epi64(y25, y1); + y26 = _mm256_add_epi64(y26, y2); + y27 = _mm256_add_epi64(y27, y3); + y23 = _mm256_mask_blend_epi64(k4, y23, y5); + y24 = _mm256_mask_blend_epi64(k4, y24, y6); + y25 = _mm256_mask_blend_epi64(k4, y25, y7); + y26 = _mm256_mask_blend_epi64(k4, y26, y8); + y27 = _mm256_mask_blend_epi64(k4, y27, y9); + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1280), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1312), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1344), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1376), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1408), y22); + /* Ai[j] = A2 + Ai[j-1] */ + r12 = (word64)(rbp); + r11 = (word64)(7); +L_ge_dsm_avx512_ifma_table: + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1280)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1312)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1344)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1376)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1408)); + /* Add */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 0)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 32)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 64)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 96)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 128)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* X3 = B - A, Y3 = B + A, Z3 = D + C, T3 = D - C */ + y23 = _mm256_permute4x64_epi64(y18, 0xf0); + y24 = _mm256_permute4x64_epi64(y19, 0xf0); + y25 = _mm256_permute4x64_epi64(y20, 0xf0); + y26 = _mm256_permute4x64_epi64(y21, 0xf0); + y27 = _mm256_permute4x64_epi64(y22, 0xf0); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y0 = _mm256_permute4x64_epi64(y18, 0xa5); + y1 = _mm256_permute4x64_epi64(y19, 0xa5); + y2 = _mm256_permute4x64_epi64(y20, 0xa5); + y3 = _mm256_permute4x64_epi64(y21, 0xa5); + y4 = _mm256_permute4x64_epi64(y22, 0xa5); + y18 = _mm256_add_epi64(y23, y0); + y19 = _mm256_add_epi64(y24, y1); + y20 = _mm256_add_epi64(y25, y2); + y21 = _mm256_add_epi64(y26, y3); + y22 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_sub_epi64(y23, y0)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_sub_epi64(y24, y1)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_sub_epi64(y25, y2)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_sub_epi64(y26, y3)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_sub_epi64(y27, y4)); + y5 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y6 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y7 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y8 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y9 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y9, y31); + y19 = _mm256_add_epi64(y19, y5); + y20 = _mm256_add_epi64(y20, y6); + y21 = _mm256_add_epi64(y21, y7); + y22 = _mm256_add_epi64(y22, y8); + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + r12 = (word64)(r12 + 0xa0); + /* To cached */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 32)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 64)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 96)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 128)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 160)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y18); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 32), y19); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 64), y20); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 96), y21); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 128), y22); + r11 = (word64)(r11 - 1); + if ((r11) != (0)) { + goto L_ge_dsm_avx512_ifma_table; + } + /* R = identity: X = 0, Y = 1, Z = 1, T = 0 */ + r9 = (word64)(1); + WC_S64(rbp, 1608) = (word64)(r9); + r9 = (word64)(0); + WC_S64(rbp, 1640) = (word64)(r9); + WC_S64(rbp, 1672) = (word64)(r9); + WC_S64(rbp, 1704) = (word64)(r9); + WC_S64(rbp, 1736) = (word64)(r9); + r9 = (word64)(1); + WC_S64(rbp, 1616) = (word64)(r9); + r9 = (word64)(0); + WC_S64(rbp, 1648) = (word64)(r9); + WC_S64(rbp, 1680) = (word64)(r9); + WC_S64(rbp, 1712) = (word64)(r9); + WC_S64(rbp, 1744) = (word64)(r9); + r9 = (word64)(0); + WC_S64(rbp, 1600) = (word64)(r9); + WC_S64(rbp, 1624) = (word64)(r9); + WC_S64(rbp, 1632) = (word64)(r9); + WC_S64(rbp, 1656) = (word64)(r9); + WC_S64(rbp, 1664) = (word64)(r9); + WC_S64(rbp, 1688) = (word64)(r9); + WC_S64(rbp, 1696) = (word64)(r9); + WC_S64(rbp, 1720) = (word64)(r9); + WC_S64(rbp, 1728) = (word64)(r9); + WC_S64(rbp, 1752) = (word64)(r9); + r10 = (word64)(0x7ffffffffffff); + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1600)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1632)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1664)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1696)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1728)); + r11 = (word64)(0xff); +L_ge_dsm_avx512_ifma_bits: + /* Double */ + y23 = _mm256_permute4x64_epi64(y18, 0x64); + y24 = _mm256_permute4x64_epi64(y19, 0x64); + y25 = _mm256_permute4x64_epi64(y20, 0x64); + y26 = _mm256_permute4x64_epi64(y21, 0x64); + y27 = _mm256_permute4x64_epi64(y22, 0x64); + y18 = _mm256_permute4x64_epi64(y18, 0x24); + y19 = _mm256_permute4x64_epi64(y19, 0x24); + y20 = _mm256_permute4x64_epi64(y20, 0x24); + y21 = _mm256_permute4x64_epi64(y21, 0x24); + y22 = _mm256_permute4x64_epi64(y22, 0x24); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_add_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* Y3 = YY + XX, Z3 = YY - XX */ + y23 = _mm256_permute4x64_epi64(y18, 0x55); + y24 = _mm256_permute4x64_epi64(y19, 0x55); + y25 = _mm256_permute4x64_epi64(y20, 0x55); + y26 = _mm256_permute4x64_epi64(y21, 0x55); + y27 = _mm256_permute4x64_epi64(y22, 0x55); + y0 = _mm256_permute4x64_epi64(y18, 0); + y1 = _mm256_permute4x64_epi64(y19, 0); + y2 = _mm256_permute4x64_epi64(y20, 0); + y3 = _mm256_permute4x64_epi64(y21, 0); + y4 = _mm256_permute4x64_epi64(y22, 0); + y5 = _mm256_add_epi64(y23, y0); + y6 = _mm256_add_epi64(y24, y1); + y7 = _mm256_add_epi64(y25, y2); + y8 = _mm256_add_epi64(y26, y3); + y9 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y30)); + y5 = _mm256_mask_blend_epi64(k2, y5, _mm256_sub_epi64(y23, y0)); + y6 = _mm256_mask_blend_epi64(k2, y6, _mm256_sub_epi64(y24, y1)); + y7 = _mm256_mask_blend_epi64(k2, y7, _mm256_sub_epi64(y25, y2)); + y8 = _mm256_mask_blend_epi64(k2, y8, _mm256_sub_epi64(y26, y3)); + y9 = _mm256_mask_blend_epi64(k2, y9, _mm256_sub_epi64(y27, y4)); + y23 = _mm256_srli_epi64(y5, 51); + y5 = _mm256_and_si256(y5, y28); + y24 = _mm256_srli_epi64(y6, 51); + y6 = _mm256_and_si256(y6, y28); + y25 = _mm256_srli_epi64(y7, 51); + y7 = _mm256_and_si256(y7, y28); + y26 = _mm256_srli_epi64(y8, 51); + y8 = _mm256_and_si256(y8, y28); + y27 = _mm256_srli_epi64(y9, 51); + y9 = _mm256_and_si256(y9, y28); + y5 = _mm256_madd52lo_epu64(y5, y27, y31); + y6 = _mm256_add_epi64(y6, y23); + y7 = _mm256_add_epi64(y7, y24); + y8 = _mm256_add_epi64(y8, y25); + y9 = _mm256_add_epi64(y9, y26); + /* X3 = AA - Y3, T3 = 2.ZZ - Z3 */ + y23 = _mm256_permute4x64_epi64(y18, 0xaf); + y24 = _mm256_permute4x64_epi64(y19, 0xaf); + y25 = _mm256_permute4x64_epi64(y20, 0xaf); + y26 = _mm256_permute4x64_epi64(y21, 0xaf); + y27 = _mm256_permute4x64_epi64(y22, 0xaf); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_sub_epi64(y23, y5)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_sub_epi64(y24, y6)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_sub_epi64(y25, y7)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_sub_epi64(y26, y8)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_sub_epi64(y27, y9)); + y0 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y1 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y2 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y3 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y4 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y4, y31); + y24 = _mm256_add_epi64(y24, y0); + y25 = _mm256_add_epi64(y25, y1); + y26 = _mm256_add_epi64(y26, y2); + y27 = _mm256_add_epi64(y27, y3); + y23 = _mm256_mask_blend_epi64(k4, y23, y5); + y24 = _mm256_mask_blend_epi64(k4, y24, y6); + y25 = _mm256_mask_blend_epi64(k4, y25, y7); + y26 = _mm256_mask_blend_epi64(k4, y26, y8); + y27 = _mm256_mask_blend_epi64(k4, y27, y9); + /* Add the multiple of A selected by this window digit */ + r12 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, r11 + 1920)); + if (((r12 & r12)) == (0)) { + goto L_ge_dsm_avx512_ifma_skip_a; + } + rsi = (word64)(r12); + rsi = (word64)((word64)((sword64)rsi >> 63)); + r12 = (word64)(r12); + r12 = (word64)(r12 ^ rsi); + r12 = (word64)(r12 - rsi); + r12 = (word64)(r12 >> 1); + r12 = (word64)(r12 * 160); + r14 = (word64)(rbp); + r14 = (word64)(r14 + r12); + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 0)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 32)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 64)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 96)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 128)); + if (((rsi & rsi)) == (0)) { + goto L_ge_dsm_avx512_ifma_noswap_a; + } + y18 = _mm256_permute4x64_epi64(y18, 0xe1); + y19 = _mm256_permute4x64_epi64(y19, 0xe1); + y20 = _mm256_permute4x64_epi64(y20, 0xe1); + y21 = _mm256_permute4x64_epi64(y21, 0xe1); + y22 = _mm256_permute4x64_epi64(y22, 0xe1); +L_ge_dsm_avx512_ifma_noswap_a: + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1760), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1792), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1824), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1856), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1888), y22); + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* Add */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1760)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1792)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1824)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1856)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1888)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* X3 = B - A, Y3 = B + A, Z3 = D + C, T3 = D - C */ + y23 = _mm256_permute4x64_epi64(y18, 0xf0); + y24 = _mm256_permute4x64_epi64(y19, 0xf0); + y25 = _mm256_permute4x64_epi64(y20, 0xf0); + y26 = _mm256_permute4x64_epi64(y21, 0xf0); + y27 = _mm256_permute4x64_epi64(y22, 0xf0); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y0 = _mm256_permute4x64_epi64(y18, 0xa5); + y1 = _mm256_permute4x64_epi64(y19, 0xa5); + y2 = _mm256_permute4x64_epi64(y20, 0xa5); + y3 = _mm256_permute4x64_epi64(y21, 0xa5); + y4 = _mm256_permute4x64_epi64(y22, 0xa5); + y18 = _mm256_add_epi64(y23, y0); + y19 = _mm256_add_epi64(y24, y1); + y20 = _mm256_add_epi64(y25, y2); + y21 = _mm256_add_epi64(y26, y3); + y22 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_sub_epi64(y23, y0)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_sub_epi64(y24, y1)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_sub_epi64(y25, y2)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_sub_epi64(y26, y3)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_sub_epi64(y27, y4)); + y5 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y6 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y7 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y8 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y9 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y9, y31); + y19 = _mm256_add_epi64(y19, y5); + y20 = _mm256_add_epi64(y20, y6); + y21 = _mm256_add_epi64(y21, y7); + y22 = _mm256_add_epi64(y22, y8); + if (((rsi & rsi)) == (0)) { + goto L_ge_dsm_avx512_ifma_nores_a; + } + y18 = _mm256_permute4x64_epi64(y18, 0xb4); + y19 = _mm256_permute4x64_epi64(y19, 0xb4); + y20 = _mm256_permute4x64_epi64(y20, 0xb4); + y21 = _mm256_permute4x64_epi64(y21, 0xb4); + y22 = _mm256_permute4x64_epi64(y22, 0xb4); +L_ge_dsm_avx512_ifma_nores_a: + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; +L_ge_dsm_avx512_ifma_skip_a: + /* Add the multiple of the base point */ + r12 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, r11 + 2176)); + if (((r12 & r12)) == (0)) { + goto L_ge_dsm_avx512_ifma_skip_b; + } + rsi = (word64)(r12); + rsi = (word64)((word64)((sword64)rsi >> 63)); + r12 = (word64)(r12); + r12 = (word64)(r12 ^ rsi); + r12 = (word64)(r12 - rsi); + r12 = (word64)(r12 >> 1); + r12 = (word64)(r12 * 96); + r14 = (word64)(rbx); + r14 = (word64)(r14 + r12); + rdx = (word64)(WC_L64(r14, 0)); + rax = (word64)(WC_L64(r14, 8)); + rcx = (word64)(WC_L64(r14, 16)); + r8 = (word64)(WC_L64(r14, 24)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1760) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1792) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1824) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1856) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1888) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 32)); + rax = (word64)(WC_L64(r14, 40)); + rcx = (word64)(WC_L64(r14, 48)); + r8 = (word64)(WC_L64(r14, 56)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1768) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1800) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1832) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1864) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1896) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 64)); + rax = (word64)(WC_L64(r14, 72)); + rcx = (word64)(WC_L64(r14, 80)); + r8 = (word64)(WC_L64(r14, 88)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1776) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1808) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1840) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1872) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1904) = (word64)(r8); + rdx = (word64)(1); + WC_S64(rbp, 1784) = (word64)(rdx); + rdx = (word64)(0); + WC_S64(rbp, 1816) = (word64)(rdx); + WC_S64(rbp, 1848) = (word64)(rdx); + WC_S64(rbp, 1880) = (word64)(rdx); + WC_S64(rbp, 1912) = (word64)(rdx); + if (((rsi & rsi)) == (0)) { + goto L_ge_dsm_avx512_ifma_noswap_b; + } + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1760)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1792)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1824)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1856)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1888)); + y18 = _mm256_permute4x64_epi64(y18, 0xe1); + y19 = _mm256_permute4x64_epi64(y19, 0xe1); + y20 = _mm256_permute4x64_epi64(y20, 0xe1); + y21 = _mm256_permute4x64_epi64(y21, 0xe1); + y22 = _mm256_permute4x64_epi64(y22, 0xe1); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1760), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1792), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1824), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1856), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1888), y22); +L_ge_dsm_avx512_ifma_noswap_b: + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* Add */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1760)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1792)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1824)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1856)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1888)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* X3 = B - A, Y3 = B + A, Z3 = D + C, T3 = D - C */ + y23 = _mm256_permute4x64_epi64(y18, 0xf0); + y24 = _mm256_permute4x64_epi64(y19, 0xf0); + y25 = _mm256_permute4x64_epi64(y20, 0xf0); + y26 = _mm256_permute4x64_epi64(y21, 0xf0); + y27 = _mm256_permute4x64_epi64(y22, 0xf0); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y0 = _mm256_permute4x64_epi64(y18, 0xa5); + y1 = _mm256_permute4x64_epi64(y19, 0xa5); + y2 = _mm256_permute4x64_epi64(y20, 0xa5); + y3 = _mm256_permute4x64_epi64(y21, 0xa5); + y4 = _mm256_permute4x64_epi64(y22, 0xa5); + y18 = _mm256_add_epi64(y23, y0); + y19 = _mm256_add_epi64(y24, y1); + y20 = _mm256_add_epi64(y25, y2); + y21 = _mm256_add_epi64(y26, y3); + y22 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_sub_epi64(y23, y0)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_sub_epi64(y24, y1)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_sub_epi64(y25, y2)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_sub_epi64(y26, y3)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_sub_epi64(y27, y4)); + y5 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y6 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y7 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y8 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y9 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y9, y31); + y19 = _mm256_add_epi64(y19, y5); + y20 = _mm256_add_epi64(y20, y6); + y21 = _mm256_add_epi64(y21, y7); + y22 = _mm256_add_epi64(y22, y8); + if (((rsi & rsi)) == (0)) { + goto L_ge_dsm_avx512_ifma_nores_b; + } + y18 = _mm256_permute4x64_epi64(y18, 0xb4); + y19 = _mm256_permute4x64_epi64(y19, 0xb4); + y20 = _mm256_permute4x64_epi64(y20, 0xb4); + y21 = _mm256_permute4x64_epi64(y21, 0xb4); + y22 = _mm256_permute4x64_epi64(y22, 0xb4); +L_ge_dsm_avx512_ifma_nores_b: + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; +L_ge_dsm_avx512_ifma_skip_b: + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_slli_epi64(y5, 4); + y10 = _mm256_slli_epi64(y5, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y5); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_slli_epi64(y6, 4); + y10 = _mm256_slli_epi64(y6, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y6); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_slli_epi64(y7, 4); + y10 = _mm256_slli_epi64(y7, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y7); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_slli_epi64(y8, 4); + y10 = _mm256_slli_epi64(y8, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y8); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_slli_epi64(y17, 4); + y10 = _mm256_slli_epi64(y17, 1); + y9 = _mm256_add_epi64(y9, y10); + y9 = _mm256_add_epi64(y9, y17); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + r11 = (word64)(r11 - 1); + if ((sword64)(r11) >= (sword64)(0)) { + goto L_ge_dsm_avx512_ifma_bits; + } + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1600), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1632), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1664), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1696), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1728), y22); + /* Convert X, Y and Z back to 4 x 64-bit field elements */ + r10 = (word64)(0x7ffffffffffff); + rdx = (word64)(WC_L64(rbp, 1600)); + rax = (word64)(WC_L64(rbp, 1632)); + rcx = (word64)(WC_L64(rbp, 1664)); + r8 = (word64)(WC_L64(rbp, 1696)); + r9 = (word64)(WC_L64(rbp, 1728)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r10); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r10); + rcx = (word64)(rcx + r11); + r11 = (word64)(rcx); + r11 = (word64)(r11 >> 51); + rcx = (word64)(rcx & r10); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r10); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 0) = (word64)(rdx); + r11 = (word64)(rcx); + r11 = (word64)(r11 << 38); + rdx = (word64)(rcx); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 8) = (word64)(r12); + r11 = (word64)(r8); + r11 = (word64)(r11 << 25); + r12 = (word64)(r8); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 16) = (word64)(rdx); + r11 = (word64)(r9); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rdi, 24) = (word64)(r12); + rdx = (word64)(WC_L64(rbp, 1608)); + rax = (word64)(WC_L64(rbp, 1640)); + rcx = (word64)(WC_L64(rbp, 1672)); + r8 = (word64)(WC_L64(rbp, 1704)); + r9 = (word64)(WC_L64(rbp, 1736)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r10); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r10); + rcx = (word64)(rcx + r11); + r11 = (word64)(rcx); + r11 = (word64)(r11 >> 51); + rcx = (word64)(rcx & r10); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r10); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(rdx); + r11 = (word64)(rcx); + r11 = (word64)(r11 << 38); + rdx = (word64)(rcx); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 40) = (word64)(r12); + r11 = (word64)(r8); + r11 = (word64)(r11 << 25); + r12 = (word64)(r8); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 48) = (word64)(rdx); + r11 = (word64)(r9); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rdi, 56) = (word64)(r12); + rdx = (word64)(WC_L64(rbp, 1616)); + rax = (word64)(WC_L64(rbp, 1648)); + rcx = (word64)(WC_L64(rbp, 1680)); + r8 = (word64)(WC_L64(rbp, 1712)); + r9 = (word64)(WC_L64(rbp, 1744)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r10); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r10); + rcx = (word64)(rcx + r11); + r11 = (word64)(rcx); + r11 = (word64)(r11 >> 51); + rcx = (word64)(rcx & r10); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r10); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 64) = (word64)(rdx); + r11 = (word64)(rcx); + r11 = (word64)(r11 << 38); + rdx = (word64)(rcx); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 72) = (word64)(r12); + r11 = (word64)(r8); + r11 = (word64)(r11 << 25); + r12 = (word64)(r8); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 80) = (word64)(rdx); + r11 = (word64)(r9); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rdi, 88) = (word64)(r12); + rax = (word64)(0); + return (int)(word32)rax; + (void)k7; +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WC_X64I_TARGET("avx512f,avx512vl,avx512ifma,avx512dq") +#if defined(HAVE_ED25519) || defined(WOLFSSL_CURVE25519_USE_ED25519) +WOLFSSL_LOCAL int ge_double_scalarmult_vartime_avx512_ifma_dq(ge_p2* r, +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + const byte* a, const ge_p3* A, const byte* b, const ge_precomp* bi, + byte* buf) +{ + word64 rdi, rsi, r14, r15, rbx, rbp, rdx, r11, rax = 0, rcx = 0, r12 = 0, + r8 = 0, r9 = 0, r10 = 0, r13 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(), + y16 = _mm256_setzero_si256(), y17 = _mm256_setzero_si256(), + y18 = _mm256_setzero_si256(), y19 = _mm256_setzero_si256(), + y20 = _mm256_setzero_si256(), y21 = _mm256_setzero_si256(), + y22 = _mm256_setzero_si256(), y23 = _mm256_setzero_si256(), + y24 = _mm256_setzero_si256(), y25 = _mm256_setzero_si256(), + y26 = _mm256_setzero_si256(), y27 = _mm256_setzero_si256(), + y28 = _mm256_setzero_si256(), y29 = _mm256_setzero_si256(), + y30 = _mm256_setzero_si256(), y31 = _mm256_setzero_si256(); + __mmask16 k1, k2, k3, k4, k5, k6, k7; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + r14 = (word64)(size_t)A; + r15 = (word64)(size_t)b; + rbx = (word64)(size_t)bi; + rbp = (word64)(size_t)buf; + + /* Window digits of the scalar multiplying A */ + /* One digit per bit of the scalar */ + rdx = (word64)(0); + r11 = (word64)(0); +L_ge_dsm_dq_a_avx512_ifma_slide_bytes: + rax = (word64)((word64)WC_L8(rsi, r11)); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 1920) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + r11 = (word64)(r11 + 1); + if ((r11) != (0x20)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_bytes; + } + /* Fold each run of digits down to one odd digit */ + r11 = (word64)(0); +L_ge_dsm_dq_a_avx512_ifma_slide_digit: + rcx = (word64)((word64)(sword64)(signed char)WC_L8(rbp, r11 + 1920)); + if (((rcx & rcx)) == (0)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_next_digit; + } + r12 = (word64)(1); +L_ge_dsm_dq_a_avx512_ifma_slide_window: + rdx = (word64)(r11); + rdx = (word64)(rdx + r12); + if ((sword64)(rdx) >= (sword64)(0x100)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_next_digit; + } + r8 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, rdx + 1920)); + if (((r8 & r8)) == (0)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_next_window; + } + /* Weight of the digit at i + b, relative to the one at i */ + r9 = (word64)(r8); + r8 = (word64)(r12); +L_ge_dsm_dq_a_avx512_ifma_slide_shift: + r9 = (word64)(r9 + r9); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_shift; + } + r9 = (word64)((word64)(sword64)(signed char)(byte)r9); + /* Fold it in if the digit at i stays within range */ + r10 = (word64)(rcx); + r10 = (word64)(r10 + r9); + if ((sword64)(r10) > (sword64)(0xf)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_sub; + } + rcx = (word64)(r10); + WC_S8(rbp, r11 + 1920) = (byte)((byte)rcx); + WC_S8(rbp, rdx + 1920) = (byte)(0); + goto L_ge_dsm_dq_a_avx512_ifma_slide_next_window; +L_ge_dsm_dq_a_avx512_ifma_slide_sub: + r10 = (word64)(rcx); + r10 = (word64)(r10 - r9); + if ((sword64)(r10) < (sword64)(-15)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_next_digit; + } + rcx = (word64)(r10); + WC_S8(rbp, r11 + 1920) = (byte)((byte)rcx); + /* Subtracting it borrows from the digits above */ +L_ge_dsm_dq_a_avx512_ifma_slide_carry: + if ((sword64)(rdx) >= (sword64)(0x100)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_next_window; + } + r10 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, rdx + 1920)); + if (((r10 & r10)) == (0)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_set; + } + WC_S8(rbp, rdx + 1920) = (byte)(0); + rdx = (word64)(rdx + 1); + goto L_ge_dsm_dq_a_avx512_ifma_slide_carry; +L_ge_dsm_dq_a_avx512_ifma_slide_set: + WC_S8(rbp, rdx + 1920) = (byte)(1); +L_ge_dsm_dq_a_avx512_ifma_slide_next_window: + r12 = (word64)(r12 + 1); + if ((sword64)(r12) <= (sword64)(6)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_window; + } +L_ge_dsm_dq_a_avx512_ifma_slide_next_digit: + r11 = (word64)(r11 + 1); + if ((sword64)(r11) < (sword64)(0x100)) { + goto L_ge_dsm_dq_a_avx512_ifma_slide_digit; + } + /* Window digits of the base point scalar */ + /* One digit per bit of the scalar */ + rdx = (word64)(0); + r11 = (word64)(0); +L_ge_dsm_dq_b_avx512_ifma_slide_bytes: + rax = (word64)((word64)WC_L8(r15, r11)); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & 1); + WC_S8(rbp, rdx + 2176) = (byte)((byte)rcx); + rax = (word64)(rax >> 1); + rdx = (word64)(rdx + 1); + r11 = (word64)(r11 + 1); + if ((r11) != (0x20)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_bytes; + } + /* Fold each run of digits down to one odd digit */ + r11 = (word64)(0); +L_ge_dsm_dq_b_avx512_ifma_slide_digit: + rcx = (word64)((word64)(sword64)(signed char)WC_L8(rbp, r11 + 2176)); + if (((rcx & rcx)) == (0)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_next_digit; + } + r12 = (word64)(1); +L_ge_dsm_dq_b_avx512_ifma_slide_window: + rdx = (word64)(r11); + rdx = (word64)(rdx + r12); + if ((sword64)(rdx) >= (sword64)(0x100)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_next_digit; + } + r8 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, rdx + 2176)); + if (((r8 & r8)) == (0)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_next_window; + } + /* Weight of the digit at i + b, relative to the one at i */ + r9 = (word64)(r8); + r8 = (word64)(r12); +L_ge_dsm_dq_b_avx512_ifma_slide_shift: + r9 = (word64)(r9 + r9); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_shift; + } + r9 = (word64)((word64)(sword64)(signed char)(byte)r9); + /* Fold it in if the digit at i stays within range */ + r10 = (word64)(rcx); + r10 = (word64)(r10 + r9); + if ((sword64)(r10) > (sword64)(0x3f)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_sub; + } + rcx = (word64)(r10); + WC_S8(rbp, r11 + 2176) = (byte)((byte)rcx); + WC_S8(rbp, rdx + 2176) = (byte)(0); + goto L_ge_dsm_dq_b_avx512_ifma_slide_next_window; +L_ge_dsm_dq_b_avx512_ifma_slide_sub: + r10 = (word64)(rcx); + r10 = (word64)(r10 - r9); + if ((sword64)(r10) < (sword64)(-63)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_next_digit; + } + rcx = (word64)(r10); + WC_S8(rbp, r11 + 2176) = (byte)((byte)rcx); + /* Subtracting it borrows from the digits above */ +L_ge_dsm_dq_b_avx512_ifma_slide_carry: + if ((sword64)(rdx) >= (sword64)(0x100)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_next_window; + } + r10 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, rdx + 2176)); + if (((r10 & r10)) == (0)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_set; + } + WC_S8(rbp, rdx + 2176) = (byte)(0); + rdx = (word64)(rdx + 1); + goto L_ge_dsm_dq_b_avx512_ifma_slide_carry; +L_ge_dsm_dq_b_avx512_ifma_slide_set: + WC_S8(rbp, rdx + 2176) = (byte)(1); +L_ge_dsm_dq_b_avx512_ifma_slide_next_window: + r12 = (word64)(r12 + 1); + if ((sword64)(r12) <= (sword64)(6)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_window; + } +L_ge_dsm_dq_b_avx512_ifma_slide_next_digit: + r11 = (word64)(r11 + 1); + if ((sword64)(r11) < (sword64)(0x100)) { + goto L_ge_dsm_dq_b_avx512_ifma_slide_digit; + } + r15 = (word64)((word64)(size_t)L_ge_ifma_consts); + y28 = _mm256_set1_epi64x((long long)WC_L64(r15, 0)); + y29 = _mm256_set1_epi64x((long long)WC_L64(r15, 8)); + y30 = _mm256_set1_epi64x((long long)WC_L64(r15, 16)); + y31 = _mm256_set1_epi64x((long long)WC_L64(r15, 24)); + r9 = (word32)(8); + k1 = (__mmask16)(word32)r9; + r9 = (word32)(0xc); + k2 = (__mmask16)(word32)r9; + r9 = (word32)(9); + k3 = (__mmask16)(word32)r9; + r9 = (word32)(6); + k4 = (__mmask16)(word32)r9; + r9 = (word32)(1); + k5 = (__mmask16)(word32)r9; + r9 = (word32)(2); + k6 = (__mmask16)(word32)r9; + r9 = (word32)(4); + k7 = (__mmask16)(word32)r9; + /* Odd multiples of A, cached and in limb form */ + r10 = (word64)(0x7ffffffffffff); + /* A in limb form: [X, Y, Z, T] */ + rdx = (word64)(WC_L64(r14, 0)); + rax = (word64)(WC_L64(r14, 8)); + rcx = (word64)(WC_L64(r14, 16)); + r8 = (word64)(WC_L64(r14, 24)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1440) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1472) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1504) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1536) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1568) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 32)); + rax = (word64)(WC_L64(r14, 40)); + rcx = (word64)(WC_L64(r14, 48)); + r8 = (word64)(WC_L64(r14, 56)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1448) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1480) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1512) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1544) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1576) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 64)); + rax = (word64)(WC_L64(r14, 72)); + rcx = (word64)(WC_L64(r14, 80)); + r8 = (word64)(WC_L64(r14, 88)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1456) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1488) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1520) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1552) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1584) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 96)); + rax = (word64)(WC_L64(r14, 104)); + rcx = (word64)(WC_L64(r14, 112)); + r8 = (word64)(WC_L64(r14, 120)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1464) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1496) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1528) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1560) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1592) = (word64)(r8); + /* Ai[0] = A */ + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1440)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1472)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1504)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1536)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1568)); + /* To cached */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 32)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 64)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 96)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 128)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 160)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 0), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 32), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 64), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 96), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 128), y22); + /* A2 = 2.A */ + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1440)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1472)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1504)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1536)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1568)); + /* Double */ + y23 = _mm256_permute4x64_epi64(y18, 0x64); + y24 = _mm256_permute4x64_epi64(y19, 0x64); + y25 = _mm256_permute4x64_epi64(y20, 0x64); + y26 = _mm256_permute4x64_epi64(y21, 0x64); + y27 = _mm256_permute4x64_epi64(y22, 0x64); + y18 = _mm256_permute4x64_epi64(y18, 0x24); + y19 = _mm256_permute4x64_epi64(y19, 0x24); + y20 = _mm256_permute4x64_epi64(y20, 0x24); + y21 = _mm256_permute4x64_epi64(y21, 0x24); + y22 = _mm256_permute4x64_epi64(y22, 0x24); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_add_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* Y3 = YY + XX, Z3 = YY - XX */ + y23 = _mm256_permute4x64_epi64(y18, 0x55); + y24 = _mm256_permute4x64_epi64(y19, 0x55); + y25 = _mm256_permute4x64_epi64(y20, 0x55); + y26 = _mm256_permute4x64_epi64(y21, 0x55); + y27 = _mm256_permute4x64_epi64(y22, 0x55); + y0 = _mm256_permute4x64_epi64(y18, 0); + y1 = _mm256_permute4x64_epi64(y19, 0); + y2 = _mm256_permute4x64_epi64(y20, 0); + y3 = _mm256_permute4x64_epi64(y21, 0); + y4 = _mm256_permute4x64_epi64(y22, 0); + y5 = _mm256_add_epi64(y23, y0); + y6 = _mm256_add_epi64(y24, y1); + y7 = _mm256_add_epi64(y25, y2); + y8 = _mm256_add_epi64(y26, y3); + y9 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y30)); + y5 = _mm256_mask_blend_epi64(k2, y5, _mm256_sub_epi64(y23, y0)); + y6 = _mm256_mask_blend_epi64(k2, y6, _mm256_sub_epi64(y24, y1)); + y7 = _mm256_mask_blend_epi64(k2, y7, _mm256_sub_epi64(y25, y2)); + y8 = _mm256_mask_blend_epi64(k2, y8, _mm256_sub_epi64(y26, y3)); + y9 = _mm256_mask_blend_epi64(k2, y9, _mm256_sub_epi64(y27, y4)); + y23 = _mm256_srli_epi64(y5, 51); + y5 = _mm256_and_si256(y5, y28); + y24 = _mm256_srli_epi64(y6, 51); + y6 = _mm256_and_si256(y6, y28); + y25 = _mm256_srli_epi64(y7, 51); + y7 = _mm256_and_si256(y7, y28); + y26 = _mm256_srli_epi64(y8, 51); + y8 = _mm256_and_si256(y8, y28); + y27 = _mm256_srli_epi64(y9, 51); + y9 = _mm256_and_si256(y9, y28); + y5 = _mm256_madd52lo_epu64(y5, y27, y31); + y6 = _mm256_add_epi64(y6, y23); + y7 = _mm256_add_epi64(y7, y24); + y8 = _mm256_add_epi64(y8, y25); + y9 = _mm256_add_epi64(y9, y26); + /* X3 = AA - Y3, T3 = 2.ZZ - Z3 */ + y23 = _mm256_permute4x64_epi64(y18, 0xaf); + y24 = _mm256_permute4x64_epi64(y19, 0xaf); + y25 = _mm256_permute4x64_epi64(y20, 0xaf); + y26 = _mm256_permute4x64_epi64(y21, 0xaf); + y27 = _mm256_permute4x64_epi64(y22, 0xaf); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_sub_epi64(y23, y5)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_sub_epi64(y24, y6)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_sub_epi64(y25, y7)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_sub_epi64(y26, y8)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_sub_epi64(y27, y9)); + y0 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y1 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y2 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y3 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y4 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y4, y31); + y24 = _mm256_add_epi64(y24, y0); + y25 = _mm256_add_epi64(y25, y1); + y26 = _mm256_add_epi64(y26, y2); + y27 = _mm256_add_epi64(y27, y3); + y23 = _mm256_mask_blend_epi64(k4, y23, y5); + y24 = _mm256_mask_blend_epi64(k4, y24, y6); + y25 = _mm256_mask_blend_epi64(k4, y25, y7); + y26 = _mm256_mask_blend_epi64(k4, y26, y8); + y27 = _mm256_mask_blend_epi64(k4, y27, y9); + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1280), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1312), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1344), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1376), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1408), y22); + /* Ai[j] = A2 + Ai[j-1] */ + r12 = (word64)(rbp); + r11 = (word64)(7); +L_ge_dsm_dq_avx512_ifma_table: + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1280)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1312)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1344)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1376)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1408)); + /* Add */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 0)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 32)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 64)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 96)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 128)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* X3 = B - A, Y3 = B + A, Z3 = D + C, T3 = D - C */ + y23 = _mm256_permute4x64_epi64(y18, 0xf0); + y24 = _mm256_permute4x64_epi64(y19, 0xf0); + y25 = _mm256_permute4x64_epi64(y20, 0xf0); + y26 = _mm256_permute4x64_epi64(y21, 0xf0); + y27 = _mm256_permute4x64_epi64(y22, 0xf0); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y0 = _mm256_permute4x64_epi64(y18, 0xa5); + y1 = _mm256_permute4x64_epi64(y19, 0xa5); + y2 = _mm256_permute4x64_epi64(y20, 0xa5); + y3 = _mm256_permute4x64_epi64(y21, 0xa5); + y4 = _mm256_permute4x64_epi64(y22, 0xa5); + y18 = _mm256_add_epi64(y23, y0); + y19 = _mm256_add_epi64(y24, y1); + y20 = _mm256_add_epi64(y25, y2); + y21 = _mm256_add_epi64(y26, y3); + y22 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_sub_epi64(y23, y0)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_sub_epi64(y24, y1)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_sub_epi64(y25, y2)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_sub_epi64(y26, y3)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_sub_epi64(y27, y4)); + y5 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y6 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y7 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y8 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y9 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y9, y31); + y19 = _mm256_add_epi64(y19, y5); + y20 = _mm256_add_epi64(y20, y6); + y21 = _mm256_add_epi64(y21, y7); + y22 = _mm256_add_epi64(y22, y8); + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + r12 = (word64)(r12 + 0xa0); + /* To cached */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 32)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 64)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 96)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 128)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(r15, 160)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 0), y18); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 32), y19); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 64), y20); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 96), y21); + _mm256_storeu_si256((__m256i*)WC_PW(r12, 128), y22); + r11 = (word64)(r11 - 1); + if ((r11) != (0)) { + goto L_ge_dsm_dq_avx512_ifma_table; + } + /* R = identity: X = 0, Y = 1, Z = 1, T = 0 */ + r9 = (word64)(1); + WC_S64(rbp, 1608) = (word64)(r9); + r9 = (word64)(0); + WC_S64(rbp, 1640) = (word64)(r9); + WC_S64(rbp, 1672) = (word64)(r9); + WC_S64(rbp, 1704) = (word64)(r9); + WC_S64(rbp, 1736) = (word64)(r9); + r9 = (word64)(1); + WC_S64(rbp, 1616) = (word64)(r9); + r9 = (word64)(0); + WC_S64(rbp, 1648) = (word64)(r9); + WC_S64(rbp, 1680) = (word64)(r9); + WC_S64(rbp, 1712) = (word64)(r9); + WC_S64(rbp, 1744) = (word64)(r9); + r9 = (word64)(0); + WC_S64(rbp, 1600) = (word64)(r9); + WC_S64(rbp, 1624) = (word64)(r9); + WC_S64(rbp, 1632) = (word64)(r9); + WC_S64(rbp, 1656) = (word64)(r9); + WC_S64(rbp, 1664) = (word64)(r9); + WC_S64(rbp, 1688) = (word64)(r9); + WC_S64(rbp, 1696) = (word64)(r9); + WC_S64(rbp, 1720) = (word64)(r9); + WC_S64(rbp, 1728) = (word64)(r9); + WC_S64(rbp, 1752) = (word64)(r9); + r10 = (word64)(0x7ffffffffffff); + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1600)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1632)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1664)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1696)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1728)); + r11 = (word64)(0xff); +L_ge_dsm_dq_avx512_ifma_bits: + /* Double */ + y23 = _mm256_permute4x64_epi64(y18, 0x64); + y24 = _mm256_permute4x64_epi64(y19, 0x64); + y25 = _mm256_permute4x64_epi64(y20, 0x64); + y26 = _mm256_permute4x64_epi64(y21, 0x64); + y27 = _mm256_permute4x64_epi64(y22, 0x64); + y18 = _mm256_permute4x64_epi64(y18, 0x24); + y19 = _mm256_permute4x64_epi64(y19, 0x24); + y20 = _mm256_permute4x64_epi64(y20, 0x24); + y21 = _mm256_permute4x64_epi64(y21, 0x24); + y22 = _mm256_permute4x64_epi64(y22, 0x24); + y18 = _mm256_mask_blend_epi64(k1, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k1, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k1, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k1, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k1, y22, _mm256_add_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* Y3 = YY + XX, Z3 = YY - XX */ + y23 = _mm256_permute4x64_epi64(y18, 0x55); + y24 = _mm256_permute4x64_epi64(y19, 0x55); + y25 = _mm256_permute4x64_epi64(y20, 0x55); + y26 = _mm256_permute4x64_epi64(y21, 0x55); + y27 = _mm256_permute4x64_epi64(y22, 0x55); + y0 = _mm256_permute4x64_epi64(y18, 0); + y1 = _mm256_permute4x64_epi64(y19, 0); + y2 = _mm256_permute4x64_epi64(y20, 0); + y3 = _mm256_permute4x64_epi64(y21, 0); + y4 = _mm256_permute4x64_epi64(y22, 0); + y5 = _mm256_add_epi64(y23, y0); + y6 = _mm256_add_epi64(y24, y1); + y7 = _mm256_add_epi64(y25, y2); + y8 = _mm256_add_epi64(y26, y3); + y9 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y30)); + y5 = _mm256_mask_blend_epi64(k2, y5, _mm256_sub_epi64(y23, y0)); + y6 = _mm256_mask_blend_epi64(k2, y6, _mm256_sub_epi64(y24, y1)); + y7 = _mm256_mask_blend_epi64(k2, y7, _mm256_sub_epi64(y25, y2)); + y8 = _mm256_mask_blend_epi64(k2, y8, _mm256_sub_epi64(y26, y3)); + y9 = _mm256_mask_blend_epi64(k2, y9, _mm256_sub_epi64(y27, y4)); + y23 = _mm256_srli_epi64(y5, 51); + y5 = _mm256_and_si256(y5, y28); + y24 = _mm256_srli_epi64(y6, 51); + y6 = _mm256_and_si256(y6, y28); + y25 = _mm256_srli_epi64(y7, 51); + y7 = _mm256_and_si256(y7, y28); + y26 = _mm256_srli_epi64(y8, 51); + y8 = _mm256_and_si256(y8, y28); + y27 = _mm256_srli_epi64(y9, 51); + y9 = _mm256_and_si256(y9, y28); + y5 = _mm256_madd52lo_epu64(y5, y27, y31); + y6 = _mm256_add_epi64(y6, y23); + y7 = _mm256_add_epi64(y7, y24); + y8 = _mm256_add_epi64(y8, y25); + y9 = _mm256_add_epi64(y9, y26); + /* X3 = AA - Y3, T3 = 2.ZZ - Z3 */ + y23 = _mm256_permute4x64_epi64(y18, 0xaf); + y24 = _mm256_permute4x64_epi64(y19, 0xaf); + y25 = _mm256_permute4x64_epi64(y20, 0xaf); + y26 = _mm256_permute4x64_epi64(y21, 0xaf); + y27 = _mm256_permute4x64_epi64(y22, 0xaf); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_sub_epi64(y23, y5)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_sub_epi64(y24, y6)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_sub_epi64(y25, y7)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_sub_epi64(y26, y8)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_sub_epi64(y27, y9)); + y0 = _mm256_srli_epi64(y23, 51); + y23 = _mm256_and_si256(y23, y28); + y1 = _mm256_srli_epi64(y24, 51); + y24 = _mm256_and_si256(y24, y28); + y2 = _mm256_srli_epi64(y25, 51); + y25 = _mm256_and_si256(y25, y28); + y3 = _mm256_srli_epi64(y26, 51); + y26 = _mm256_and_si256(y26, y28); + y4 = _mm256_srli_epi64(y27, 51); + y27 = _mm256_and_si256(y27, y28); + y23 = _mm256_madd52lo_epu64(y23, y4, y31); + y24 = _mm256_add_epi64(y24, y0); + y25 = _mm256_add_epi64(y25, y1); + y26 = _mm256_add_epi64(y26, y2); + y27 = _mm256_add_epi64(y27, y3); + y23 = _mm256_mask_blend_epi64(k4, y23, y5); + y24 = _mm256_mask_blend_epi64(k4, y24, y6); + y25 = _mm256_mask_blend_epi64(k4, y25, y7); + y26 = _mm256_mask_blend_epi64(k4, y26, y8); + y27 = _mm256_mask_blend_epi64(k4, y27, y9); + /* Add the multiple of A selected by this window digit */ + r12 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, r11 + 1920)); + if (((r12 & r12)) == (0)) { + goto L_ge_dsm_dq_avx512_ifma_skip_a; + } + rsi = (word64)(r12); + rsi = (word64)((word64)((sword64)rsi >> 63)); + r12 = (word64)(r12); + r12 = (word64)(r12 ^ rsi); + r12 = (word64)(r12 - rsi); + r12 = (word64)(r12 >> 1); + r12 = (word64)(r12 * 160); + r14 = (word64)(rbp); + r14 = (word64)(r14 + r12); + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 0)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 32)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 64)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 96)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 128)); + if (((rsi & rsi)) == (0)) { + goto L_ge_dsm_dq_avx512_ifma_noswap_a; + } + y18 = _mm256_permute4x64_epi64(y18, 0xe1); + y19 = _mm256_permute4x64_epi64(y19, 0xe1); + y20 = _mm256_permute4x64_epi64(y20, 0xe1); + y21 = _mm256_permute4x64_epi64(y21, 0xe1); + y22 = _mm256_permute4x64_epi64(y22, 0xe1); +L_ge_dsm_dq_avx512_ifma_noswap_a: + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1760), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1792), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1824), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1856), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1888), y22); + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* Add */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1760)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1792)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1824)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1856)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1888)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* X3 = B - A, Y3 = B + A, Z3 = D + C, T3 = D - C */ + y23 = _mm256_permute4x64_epi64(y18, 0xf0); + y24 = _mm256_permute4x64_epi64(y19, 0xf0); + y25 = _mm256_permute4x64_epi64(y20, 0xf0); + y26 = _mm256_permute4x64_epi64(y21, 0xf0); + y27 = _mm256_permute4x64_epi64(y22, 0xf0); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y0 = _mm256_permute4x64_epi64(y18, 0xa5); + y1 = _mm256_permute4x64_epi64(y19, 0xa5); + y2 = _mm256_permute4x64_epi64(y20, 0xa5); + y3 = _mm256_permute4x64_epi64(y21, 0xa5); + y4 = _mm256_permute4x64_epi64(y22, 0xa5); + y18 = _mm256_add_epi64(y23, y0); + y19 = _mm256_add_epi64(y24, y1); + y20 = _mm256_add_epi64(y25, y2); + y21 = _mm256_add_epi64(y26, y3); + y22 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_sub_epi64(y23, y0)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_sub_epi64(y24, y1)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_sub_epi64(y25, y2)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_sub_epi64(y26, y3)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_sub_epi64(y27, y4)); + y5 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y6 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y7 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y8 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y9 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y9, y31); + y19 = _mm256_add_epi64(y19, y5); + y20 = _mm256_add_epi64(y20, y6); + y21 = _mm256_add_epi64(y21, y7); + y22 = _mm256_add_epi64(y22, y8); + if (((rsi & rsi)) == (0)) { + goto L_ge_dsm_dq_avx512_ifma_nores_a; + } + y18 = _mm256_permute4x64_epi64(y18, 0xb4); + y19 = _mm256_permute4x64_epi64(y19, 0xb4); + y20 = _mm256_permute4x64_epi64(y20, 0xb4); + y21 = _mm256_permute4x64_epi64(y21, 0xb4); + y22 = _mm256_permute4x64_epi64(y22, 0xb4); +L_ge_dsm_dq_avx512_ifma_nores_a: + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; +L_ge_dsm_dq_avx512_ifma_skip_a: + /* Add the multiple of the base point */ + r12 = (word64)((word64)(sword64)(signed char)WC_L8(rbp, r11 + 2176)); + if (((r12 & r12)) == (0)) { + goto L_ge_dsm_dq_avx512_ifma_skip_b; + } + rsi = (word64)(r12); + rsi = (word64)((word64)((sword64)rsi >> 63)); + r12 = (word64)(r12); + r12 = (word64)(r12 ^ rsi); + r12 = (word64)(r12 - rsi); + r12 = (word64)(r12 >> 1); + r12 = (word64)(r12 * 96); + r14 = (word64)(rbx); + r14 = (word64)(r14 + r12); + rdx = (word64)(WC_L64(r14, 0)); + rax = (word64)(WC_L64(r14, 8)); + rcx = (word64)(WC_L64(r14, 16)); + r8 = (word64)(WC_L64(r14, 24)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1760) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1792) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1824) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1856) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1888) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 32)); + rax = (word64)(WC_L64(r14, 40)); + rcx = (word64)(WC_L64(r14, 48)); + r8 = (word64)(WC_L64(r14, 56)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1768) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1800) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1832) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1864) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1896) = (word64)(r8); + rdx = (word64)(WC_L64(r14, 64)); + rax = (word64)(WC_L64(r14, 72)); + rcx = (word64)(WC_L64(r14, 80)); + r8 = (word64)(WC_L64(r14, 88)); + r9 = (word64)(r8); + r9 = (word64)(r9 >> 63); + r9 = (word64)(r9 * 19); + r8 = (word64)(r8 << 1); + r8 = (word64)(r8 >> 1); + r13 = (word64)(rdx); + r13 = (word64)(r13 & r10); + r13 = (word64)(r13 + r9); + WC_S64(rbp, 1776) = (word64)(r13); + rdx = (word64)((rdx >> 51) | (rax << 13)); + r9 = (word64)(rdx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1808) = (word64)(r9); + rax = (word64)((rax >> 38) | (rcx << 26)); + r9 = (word64)(rax); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1840) = (word64)(r9); + rcx = (word64)((rcx >> 25) | (r8 << 39)); + r9 = (word64)(rcx); + r9 = (word64)(r9 & r10); + WC_S64(rbp, 1872) = (word64)(r9); + r8 = (word64)(r8 >> 12); + WC_S64(rbp, 1904) = (word64)(r8); + rdx = (word64)(1); + WC_S64(rbp, 1784) = (word64)(rdx); + rdx = (word64)(0); + WC_S64(rbp, 1816) = (word64)(rdx); + WC_S64(rbp, 1848) = (word64)(rdx); + WC_S64(rbp, 1880) = (word64)(rdx); + WC_S64(rbp, 1912) = (word64)(rdx); + if (((rsi & rsi)) == (0)) { + goto L_ge_dsm_dq_avx512_ifma_noswap_b; + } + y18 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1760)); + y19 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1792)); + y20 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1824)); + y21 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1856)); + y22 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1888)); + y18 = _mm256_permute4x64_epi64(y18, 0xe1); + y19 = _mm256_permute4x64_epi64(y19, 0xe1); + y20 = _mm256_permute4x64_epi64(y20, 0xe1); + y21 = _mm256_permute4x64_epi64(y21, 0xe1); + y22 = _mm256_permute4x64_epi64(y22, 0xe1); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1760), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1792), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1824), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1856), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1888), y22); +L_ge_dsm_dq_avx512_ifma_noswap_b: + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* Add */ + y23 = _mm256_permute4x64_epi64(y18, 0xb0); + y24 = _mm256_permute4x64_epi64(y19, 0xb0); + y25 = _mm256_permute4x64_epi64(y20, 0xb0); + y26 = _mm256_permute4x64_epi64(y21, 0xb0); + y27 = _mm256_permute4x64_epi64(y22, 0xb0); + y18 = _mm256_permute4x64_epi64(y18, 0xb5); + y19 = _mm256_permute4x64_epi64(y19, 0xb5); + y20 = _mm256_permute4x64_epi64(y20, 0xb5); + y21 = _mm256_permute4x64_epi64(y21, 0xb5); + y22 = _mm256_permute4x64_epi64(y22, 0xb5); + y18 = _mm256_mask_blend_epi64(k5, y18, _mm256_add_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k5, y19, _mm256_add_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k5, y20, _mm256_add_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k5, y21, _mm256_add_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k5, y22, _mm256_add_epi64(y22, y27)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_add_epi64(y18, y29)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_add_epi64(y19, y30)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_add_epi64(y20, y30)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_add_epi64(y21, y30)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_add_epi64(y22, y30)); + y18 = _mm256_mask_blend_epi64(k6, y18, _mm256_sub_epi64(y18, y23)); + y19 = _mm256_mask_blend_epi64(k6, y19, _mm256_sub_epi64(y19, y24)); + y20 = _mm256_mask_blend_epi64(k6, y20, _mm256_sub_epi64(y20, y25)); + y21 = _mm256_mask_blend_epi64(k6, y21, _mm256_sub_epi64(y21, y26)); + y22 = _mm256_mask_blend_epi64(k6, y22, _mm256_sub_epi64(y22, y27)); + y0 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y1 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y2 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y3 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y4 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y4, y31); + y19 = _mm256_add_epi64(y19, y0); + y20 = _mm256_add_epi64(y20, y1); + y21 = _mm256_add_epi64(y21, y2); + y22 = _mm256_add_epi64(y22, y3); + y23 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1760)); + y24 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1792)); + y25 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1824)); + y26 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1856)); + y27 = _mm256_loadu_si256((const __m256i*)WC_PR(rbp, 1888)); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + /* X3 = B - A, Y3 = B + A, Z3 = D + C, T3 = D - C */ + y23 = _mm256_permute4x64_epi64(y18, 0xf0); + y24 = _mm256_permute4x64_epi64(y19, 0xf0); + y25 = _mm256_permute4x64_epi64(y20, 0xf0); + y26 = _mm256_permute4x64_epi64(y21, 0xf0); + y27 = _mm256_permute4x64_epi64(y22, 0xf0); + y23 = _mm256_mask_blend_epi64(k2, y23, _mm256_add_epi64(y23, y23)); + y24 = _mm256_mask_blend_epi64(k2, y24, _mm256_add_epi64(y24, y24)); + y25 = _mm256_mask_blend_epi64(k2, y25, _mm256_add_epi64(y25, y25)); + y26 = _mm256_mask_blend_epi64(k2, y26, _mm256_add_epi64(y26, y26)); + y27 = _mm256_mask_blend_epi64(k2, y27, _mm256_add_epi64(y27, y27)); + y0 = _mm256_permute4x64_epi64(y18, 0xa5); + y1 = _mm256_permute4x64_epi64(y19, 0xa5); + y2 = _mm256_permute4x64_epi64(y20, 0xa5); + y3 = _mm256_permute4x64_epi64(y21, 0xa5); + y4 = _mm256_permute4x64_epi64(y22, 0xa5); + y18 = _mm256_add_epi64(y23, y0); + y19 = _mm256_add_epi64(y24, y1); + y20 = _mm256_add_epi64(y25, y2); + y21 = _mm256_add_epi64(y26, y3); + y22 = _mm256_add_epi64(y27, y4); + y23 = _mm256_mask_blend_epi64(k3, y23, _mm256_add_epi64(y23, y29)); + y24 = _mm256_mask_blend_epi64(k3, y24, _mm256_add_epi64(y24, y30)); + y25 = _mm256_mask_blend_epi64(k3, y25, _mm256_add_epi64(y25, y30)); + y26 = _mm256_mask_blend_epi64(k3, y26, _mm256_add_epi64(y26, y30)); + y27 = _mm256_mask_blend_epi64(k3, y27, _mm256_add_epi64(y27, y30)); + y18 = _mm256_mask_blend_epi64(k3, y18, _mm256_sub_epi64(y23, y0)); + y19 = _mm256_mask_blend_epi64(k3, y19, _mm256_sub_epi64(y24, y1)); + y20 = _mm256_mask_blend_epi64(k3, y20, _mm256_sub_epi64(y25, y2)); + y21 = _mm256_mask_blend_epi64(k3, y21, _mm256_sub_epi64(y26, y3)); + y22 = _mm256_mask_blend_epi64(k3, y22, _mm256_sub_epi64(y27, y4)); + y5 = _mm256_srli_epi64(y18, 51); + y18 = _mm256_and_si256(y18, y28); + y6 = _mm256_srli_epi64(y19, 51); + y19 = _mm256_and_si256(y19, y28); + y7 = _mm256_srli_epi64(y20, 51); + y20 = _mm256_and_si256(y20, y28); + y8 = _mm256_srli_epi64(y21, 51); + y21 = _mm256_and_si256(y21, y28); + y9 = _mm256_srli_epi64(y22, 51); + y22 = _mm256_and_si256(y22, y28); + y18 = _mm256_madd52lo_epu64(y18, y9, y31); + y19 = _mm256_add_epi64(y19, y5); + y20 = _mm256_add_epi64(y20, y6); + y21 = _mm256_add_epi64(y21, y7); + y22 = _mm256_add_epi64(y22, y8); + if (((rsi & rsi)) == (0)) { + goto L_ge_dsm_dq_avx512_ifma_nores_b; + } + y18 = _mm256_permute4x64_epi64(y18, 0xb4); + y19 = _mm256_permute4x64_epi64(y19, 0xb4); + y20 = _mm256_permute4x64_epi64(y20, 0xb4); + y21 = _mm256_permute4x64_epi64(y21, 0xb4); + y22 = _mm256_permute4x64_epi64(y22, 0xb4); +L_ge_dsm_dq_avx512_ifma_nores_b: + y23 = y18; + y24 = y19; + y25 = y20; + y26 = y21; + y27 = y22; +L_ge_dsm_dq_avx512_ifma_skip_b: + /* To p3 */ + y18 = _mm256_permute4x64_epi64(y23, 0x24); + y19 = _mm256_permute4x64_epi64(y24, 0x24); + y20 = _mm256_permute4x64_epi64(y25, 0x24); + y21 = _mm256_permute4x64_epi64(y26, 0x24); + y22 = _mm256_permute4x64_epi64(y27, 0x24); + y23 = _mm256_permute4x64_epi64(y23, 0x7b); + y24 = _mm256_permute4x64_epi64(y24, 0x7b); + y25 = _mm256_permute4x64_epi64(y25, 0x7b); + y26 = _mm256_permute4x64_epi64(y26, 0x7b); + y27 = _mm256_permute4x64_epi64(y27, 0x7b); + /* Multiply 4 field elements */ + y0 = _mm256_setzero_si256(); + y9 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y10 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y11 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y12 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y13 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y14 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y15 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + y16 = _mm256_setzero_si256(); + y8 = _mm256_setzero_si256(); + y17 = _mm256_setzero_si256(); + y0 = _mm256_madd52lo_epu64(y0, y18, y23); + y9 = _mm256_madd52hi_epu64(y9, y18, y23); + y1 = _mm256_madd52lo_epu64(y1, y18, y24); + y10 = _mm256_madd52hi_epu64(y10, y18, y24); + y2 = _mm256_madd52lo_epu64(y2, y18, y25); + y11 = _mm256_madd52hi_epu64(y11, y18, y25); + y3 = _mm256_madd52lo_epu64(y3, y18, y26); + y12 = _mm256_madd52hi_epu64(y12, y18, y26); + y4 = _mm256_madd52lo_epu64(y4, y18, y27); + y13 = _mm256_madd52hi_epu64(y13, y18, y27); + y1 = _mm256_madd52lo_epu64(y1, y19, y23); + y10 = _mm256_madd52hi_epu64(y10, y19, y23); + y2 = _mm256_madd52lo_epu64(y2, y19, y24); + y11 = _mm256_madd52hi_epu64(y11, y19, y24); + y3 = _mm256_madd52lo_epu64(y3, y19, y25); + y12 = _mm256_madd52hi_epu64(y12, y19, y25); + y4 = _mm256_madd52lo_epu64(y4, y19, y26); + y13 = _mm256_madd52hi_epu64(y13, y19, y26); + y5 = _mm256_madd52lo_epu64(y5, y19, y27); + y14 = _mm256_madd52hi_epu64(y14, y19, y27); + y2 = _mm256_madd52lo_epu64(y2, y20, y23); + y11 = _mm256_madd52hi_epu64(y11, y20, y23); + y3 = _mm256_madd52lo_epu64(y3, y20, y24); + y12 = _mm256_madd52hi_epu64(y12, y20, y24); + y4 = _mm256_madd52lo_epu64(y4, y20, y25); + y13 = _mm256_madd52hi_epu64(y13, y20, y25); + y5 = _mm256_madd52lo_epu64(y5, y20, y26); + y14 = _mm256_madd52hi_epu64(y14, y20, y26); + y6 = _mm256_madd52lo_epu64(y6, y20, y27); + y15 = _mm256_madd52hi_epu64(y15, y20, y27); + y3 = _mm256_madd52lo_epu64(y3, y21, y23); + y12 = _mm256_madd52hi_epu64(y12, y21, y23); + y4 = _mm256_madd52lo_epu64(y4, y21, y24); + y13 = _mm256_madd52hi_epu64(y13, y21, y24); + y5 = _mm256_madd52lo_epu64(y5, y21, y25); + y14 = _mm256_madd52hi_epu64(y14, y21, y25); + y6 = _mm256_madd52lo_epu64(y6, y21, y26); + y15 = _mm256_madd52hi_epu64(y15, y21, y26); + y7 = _mm256_madd52lo_epu64(y7, y21, y27); + y16 = _mm256_madd52hi_epu64(y16, y21, y27); + y4 = _mm256_madd52lo_epu64(y4, y22, y23); + y13 = _mm256_madd52hi_epu64(y13, y22, y23); + y5 = _mm256_madd52lo_epu64(y5, y22, y24); + y14 = _mm256_madd52hi_epu64(y14, y22, y24); + y6 = _mm256_madd52lo_epu64(y6, y22, y25); + y15 = _mm256_madd52hi_epu64(y15, y22, y25); + y7 = _mm256_madd52lo_epu64(y7, y22, y26); + y16 = _mm256_madd52hi_epu64(y16, y22, y26); + y8 = _mm256_madd52lo_epu64(y8, y22, y27); + y17 = _mm256_madd52hi_epu64(y17, y22, y27); + y9 = _mm256_add_epi64(y9, y9); + y1 = _mm256_add_epi64(y1, y9); + y10 = _mm256_add_epi64(y10, y10); + y2 = _mm256_add_epi64(y2, y10); + y11 = _mm256_add_epi64(y11, y11); + y3 = _mm256_add_epi64(y3, y11); + y12 = _mm256_add_epi64(y12, y12); + y4 = _mm256_add_epi64(y4, y12); + y13 = _mm256_add_epi64(y13, y13); + y5 = _mm256_add_epi64(y5, y13); + y14 = _mm256_add_epi64(y14, y14); + y6 = _mm256_add_epi64(y6, y14); + y15 = _mm256_add_epi64(y15, y15); + y7 = _mm256_add_epi64(y7, y15); + y16 = _mm256_add_epi64(y16, y16); + y8 = _mm256_add_epi64(y8, y16); + y17 = _mm256_add_epi64(y17, y17); + /* Reduce */ + y9 = _mm256_mullo_epi64(y5, y31); + y0 = _mm256_add_epi64(y0, y9); + y9 = _mm256_mullo_epi64(y6, y31); + y1 = _mm256_add_epi64(y1, y9); + y9 = _mm256_mullo_epi64(y7, y31); + y2 = _mm256_add_epi64(y2, y9); + y9 = _mm256_mullo_epi64(y8, y31); + y3 = _mm256_add_epi64(y3, y9); + y9 = _mm256_mullo_epi64(y17, y31); + y4 = _mm256_add_epi64(y4, y9); + y23 = _mm256_srli_epi64(y0, 51); + y0 = _mm256_and_si256(y0, y28); + y24 = _mm256_srli_epi64(y1, 51); + y1 = _mm256_and_si256(y1, y28); + y25 = _mm256_srli_epi64(y2, 51); + y2 = _mm256_and_si256(y2, y28); + y26 = _mm256_srli_epi64(y3, 51); + y3 = _mm256_and_si256(y3, y28); + y27 = _mm256_srli_epi64(y4, 51); + y4 = _mm256_and_si256(y4, y28); + y18 = y0; + y18 = _mm256_madd52lo_epu64(y18, y27, y31); + y19 = _mm256_add_epi64(y1, y23); + y20 = _mm256_add_epi64(y2, y24); + y21 = _mm256_add_epi64(y3, y25); + y22 = _mm256_add_epi64(y4, y26); + r11 = (word64)(r11 - 1); + if ((sword64)(r11) >= (sword64)(0)) { + goto L_ge_dsm_dq_avx512_ifma_bits; + } + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1600), y18); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1632), y19); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1664), y20); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1696), y21); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 1728), y22); + /* Convert X, Y and Z back to 4 x 64-bit field elements */ + r10 = (word64)(0x7ffffffffffff); + rdx = (word64)(WC_L64(rbp, 1600)); + rax = (word64)(WC_L64(rbp, 1632)); + rcx = (word64)(WC_L64(rbp, 1664)); + r8 = (word64)(WC_L64(rbp, 1696)); + r9 = (word64)(WC_L64(rbp, 1728)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r10); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r10); + rcx = (word64)(rcx + r11); + r11 = (word64)(rcx); + r11 = (word64)(r11 >> 51); + rcx = (word64)(rcx & r10); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r10); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 0) = (word64)(rdx); + r11 = (word64)(rcx); + r11 = (word64)(r11 << 38); + rdx = (word64)(rcx); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 8) = (word64)(r12); + r11 = (word64)(r8); + r11 = (word64)(r11 << 25); + r12 = (word64)(r8); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 16) = (word64)(rdx); + r11 = (word64)(r9); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rdi, 24) = (word64)(r12); + rdx = (word64)(WC_L64(rbp, 1608)); + rax = (word64)(WC_L64(rbp, 1640)); + rcx = (word64)(WC_L64(rbp, 1672)); + r8 = (word64)(WC_L64(rbp, 1704)); + r9 = (word64)(WC_L64(rbp, 1736)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r10); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r10); + rcx = (word64)(rcx + r11); + r11 = (word64)(rcx); + r11 = (word64)(r11 >> 51); + rcx = (word64)(rcx & r10); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r10); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(rdx); + r11 = (word64)(rcx); + r11 = (word64)(r11 << 38); + rdx = (word64)(rcx); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 40) = (word64)(r12); + r11 = (word64)(r8); + r11 = (word64)(r11 << 25); + r12 = (word64)(r8); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 48) = (word64)(rdx); + r11 = (word64)(r9); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rdi, 56) = (word64)(r12); + rdx = (word64)(WC_L64(rbp, 1616)); + rax = (word64)(WC_L64(rbp, 1648)); + rcx = (word64)(WC_L64(rbp, 1680)); + r8 = (word64)(WC_L64(rbp, 1712)); + r9 = (word64)(WC_L64(rbp, 1744)); + r11 = (word64)(rdx); + r11 = (word64)(r11 >> 51); + rdx = (word64)(rdx & r10); + rax = (word64)(rax + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 >> 51); + rax = (word64)(rax & r10); + rcx = (word64)(rcx + r11); + r11 = (word64)(rcx); + r11 = (word64)(r11 >> 51); + rcx = (word64)(rcx & r10); + r8 = (word64)(r8 + r11); + r11 = (word64)(r8); + r11 = (word64)(r11 >> 51); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 + r11); + r11 = (word64)(r9); + r11 = (word64)(r11 >> 51); + r9 = (word64)(r9 & r10); + r11 = (word64)(r11 * 19); + rdx = (word64)(rdx + r11); + r11 = (word64)(rax); + r11 = (word64)(r11 << 51); + r12 = (word64)(rax); + r12 = (word64)(r12 >> 13); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 64) = (word64)(rdx); + r11 = (word64)(rcx); + r11 = (word64)(r11 << 38); + rdx = (word64)(rcx); + rdx = (word64)(rdx >> 26); + cf = _addcarry_u64(0, r12, r11, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 72) = (word64)(r12); + r11 = (word64)(r8); + r11 = (word64)(r11 << 25); + r12 = (word64)(r8); + r12 = (word64)(r12 >> 39); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdi, 80) = (word64)(rdx); + r11 = (word64)(r9); + r11 = (word64)(r11 << 12); + r12 = (word64)(r12 + r11); + WC_S64(rdi, 88) = (word64)(r12); + rax = (word64)(0); + return (int)(word32)rax; + (void)k7; +} + +#endif /* HAVE_ED25519 || WOLFSSL_CURVE25519_USE_ED25519 */ + +#endif /* HAVE_ED25519 */ +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* HAVE_INTEL_AVX2 */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/poly1305_intrin.c b/wolfcrypt/src/poly1305_intrin.c new file mode 100644 index 00000000000..dff2360869f --- /dev/null +++ b/wolfcrypt/src/poly1305_intrin.c @@ -0,0 +1,3677 @@ +/* poly1305_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define _WC_BUILDING_POLY1305_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_X86_64_BUILD +#ifdef HAVE_INTEL_AVX1 +extern WOLFSSL_LOCAL void poly1305_setkey_avx(Poly1305* ctx, const byte* key); +extern WOLFSSL_LOCAL void poly1305_block_avx(Poly1305* ctx, + const unsigned char* m); +extern WOLFSSL_LOCAL void poly1305_blocks_avx(Poly1305* ctx, + const unsigned char* m, size_t bytes); +extern WOLFSSL_LOCAL void poly1305_final_avx(Poly1305* ctx, byte* mac); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void poly1305_calc_powers_avx2(Poly1305* ctx); +extern WOLFSSL_LOCAL void poly1305_setkey_avx2(Poly1305* ctx, const byte* key); +extern WOLFSSL_LOCAL void poly1305_blocks_avx2(Poly1305* ctx, + const unsigned char* m, size_t bytes); +extern WOLFSSL_LOCAL void poly1305_final_avx2(Poly1305* ctx, byte* mac); +#endif +#ifdef HAVE_INTEL_AVX512 +extern WOLFSSL_LOCAL void poly1305_calc_powers_avx512(Poly1305* ctx); +extern WOLFSSL_LOCAL void poly1305_blocks_avx512(Poly1305* ctx, + const unsigned char* m, size_t bytes); +extern WOLFSSL_LOCAL void poly1305_final_avx512(Poly1305* ctx, byte* mac); +extern WOLFSSL_LOCAL void poly1305_calc_powers_avx512ifma(Poly1305* ctx); +extern WOLFSSL_LOCAL void poly1305_setkey_avx512ifma(Poly1305* ctx, + const byte* key); +extern WOLFSSL_LOCAL void poly1305_blocks_avx512ifma(Poly1305* ctx, + const unsigned char* m, size_t bytes); +extern WOLFSSL_LOCAL void poly1305_final_avx512ifma(Poly1305* ctx, byte* mac); +extern WOLFSSL_LOCAL void poly1305_fold_avx512ifma(Poly1305* ctx, + word32 nBlocks); + +#endif +#endif +#ifdef WOLFSSL_X86_64_BUILD +#ifdef HAVE_INTEL_AVX1 +WOLFSSL_LOCAL void poly1305_setkey_avx(Poly1305* ctx, const byte* key) +{ + word64 rdi, rsi, r10, r11, rdx, rax, rcx, r8, r9; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)key; + + r10 = (word64)(0xffffffc0fffffffULL); + r11 = (word64)(0xffffffc0ffffffcULL); + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(rdx & r10); + rax = (word64)(rax & r11); + r10 = (word64)(rdx); + r11 = (word64)(rax); + r9 = (word64)(0); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r9); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r9); + WC_S64(rdi, 48) = (word64)(rcx); + WC_S64(rdi, 56) = (word64)(r8); + WC_S64(rdi, 352) = (word64)(r9); + WC_S64(rdi, 408) = (word64)(r9); + WC_S64(rdi, 360) = (word64)(rdx); + WC_S64(rdi, 416) = (word64)(rax); + r10 = (word64)(r10 + rdx); + r11 = (word64)(r11 + rax); + WC_S64(rdi, 368) = (word64)(r10); + WC_S64(rdi, 424) = (word64)(r11); + r10 = (word64)(r10 + rdx); + r11 = (word64)(r11 + rax); + WC_S64(rdi, 376) = (word64)(r10); + WC_S64(rdi, 432) = (word64)(r11); + r10 = (word64)(r10 + rdx); + r11 = (word64)(r11 + rax); + WC_S64(rdi, 384) = (word64)(r10); + WC_S64(rdi, 440) = (word64)(r11); + r10 = (word64)(r10 + rdx); + r11 = (word64)(r11 + rax); + WC_S64(rdi, 392) = (word64)(r10); + WC_S64(rdi, 448) = (word64)(r11); + r10 = (word64)(r10 + rdx); + r11 = (word64)(r11 + rax); + WC_S64(rdi, 400) = (word64)(r10); + WC_S64(rdi, 456) = (word64)(r11); + WC_S64(rdi, 608) = (word64)(r9); + WC_S8(rdi, 616) = (byte)(1); +} + +WOLFSSL_LOCAL void poly1305_block_avx(Poly1305* ctx, const unsigned char* m) +{ + word64 rdi, rsi, r15, rbx, r8, r9, r10, r14, r11, r12, rax, rdx = 0, r13; + unsigned char cf; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + + r15 = (word64)(WC_L64(rdi, 0)); + rbx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)(WC_L64(rdi, 24)); + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(0); + r14 = (r14 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, 616)) & 0xff); + /* h += m */ + r11 = (word64)(WC_L64(rsi, 0)); + r12 = (word64)(WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + rax = (word64)(rbx); + cf = _addcarry_u64(cf, r10, r14, (unsigned long long*)&r10); + /* r[1] * h[0] => rdx, rax ==> t2, t1 */ + WC_X64I_MUL128(rax, rdx, rax, r8); + r12 = (word64)(rax); + r13 = (word64)(rdx); + /* r[0] * h[1] => rdx, rax ++> t2, t1 */ + rax = (word64)(r15); + WC_X64I_MUL128(rax, rdx, rax, r9); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + rax = (word64)(r15); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* r[0] * h[0] => rdx, rax ==> t4, t0 */ + WC_X64I_MUL128(rax, rdx, rax, r8); + r11 = (word64)(rax); + r8 = (word64)(rdx); + /* r[1] * h[1] => rdx, rax =+> t3, t2 */ + rax = (word64)(rbx); + WC_X64I_MUL128(rax, rdx, rax, r9); + /* r[0] * h[2] +> t2 */ + r13 = (word64)(r13 + WC_L64(rdi, r10 * 8 + 352)); + r14 = (word64)(rdx); + cf = _addcarry_u64(0, r12, r8, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rax, (unsigned long long*)&r13); + /* r[1] * h[2] +> t3 */ + cf = _addcarry_u64(cf, r14, WC_L64(rdi, r10 * 8 + 408), ( + unsigned long long*)&r14); + /* r * h in r14, r13, r12, r11 */ + /* h = (r * h) mod 2^130 - 5 */ + r10 = (word64)(r13); + r13 = (word64)(r13 & -4); + r10 = (word64)(r10 & 3); + cf = _addcarry_u64(0, r11, r13, (unsigned long long*)&r11); + r8 = (word64)(r13); + cf = _addcarry_u64(cf, r12, r14, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r8 = (word64)((r8 >> 2) | (r14 << 62)); + r14 = (word64)(r14 >> 2); + cf = _addcarry_u64(0, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r12, r14, (unsigned long long*)&r12); + r9 = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* h in r10, r9, r8 */ + /* Store h to ctx */ + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); +} + +WOLFSSL_LOCAL void poly1305_blocks_avx(Poly1305* ctx, const unsigned char* m, + size_t bytes) +{ + word64 rdi, rsi, rcx, r15, rbx, r8, r9, r10, r11 = 0, r12 = 0, rax = 0, + rdx = 0, r13 = 0, r14 = 0; + unsigned char cf; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)bytes; + + r15 = (word64)(WC_L64(rdi, 0)); + rbx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)(WC_L64(rdi, 24)); + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rdi, 40)); +L_poly1305_avx_blocks_start: + /* h += m */ + r11 = (word64)(WC_L64(rsi, 0)); + r12 = (word64)(WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + rax = (word64)(rbx); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* r[1] * h[0] => rdx, rax ==> t2, t1 */ + WC_X64I_MUL128(rax, rdx, rax, r8); + r12 = (word64)(rax); + r13 = (word64)(rdx); + /* r[0] * h[1] => rdx, rax ++> t2, t1 */ + rax = (word64)(r15); + WC_X64I_MUL128(rax, rdx, rax, r9); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + rax = (word64)(r15); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* r[0] * h[0] => rdx, rax ==> t4, t0 */ + WC_X64I_MUL128(rax, rdx, rax, r8); + r11 = (word64)(rax); + r8 = (word64)(rdx); + /* r[1] * h[1] => rdx, rax =+> t3, t2 */ + rax = (word64)(rbx); + WC_X64I_MUL128(rax, rdx, rax, r9); + /* r[0] * h[2] +> t2 */ + r13 = (word64)(r13 + WC_L64(rdi, r10 * 8 + 360)); + r14 = (word64)(rdx); + cf = _addcarry_u64(0, r12, r8, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rax, (unsigned long long*)&r13); + /* r[1] * h[2] +> t3 */ + cf = _addcarry_u64(cf, r14, WC_L64(rdi, r10 * 8 + 416), ( + unsigned long long*)&r14); + /* r * h in r14, r13, r12, r11 */ + /* h = (r * h) mod 2^130 - 5 */ + r10 = (word64)(r13); + r13 = (word64)(r13 & -4); + r10 = (word64)(r10 & 3); + cf = _addcarry_u64(0, r11, r13, (unsigned long long*)&r11); + r8 = (word64)(r13); + cf = _addcarry_u64(cf, r12, r14, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r8 = (word64)((r8 >> 2) | (r14 << 62)); + r14 = (word64)(r14 >> 2); + cf = _addcarry_u64(0, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r12, r14, (unsigned long long*)&r12); + r9 = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* h in r10, r9, r8 */ + /* Next block from message */ + rsi = (word64)(rsi + 0x10); + rcx = (word64)(rcx - 0x10); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_poly1305_avx_blocks_start; + } + /* Store h to ctx */ + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); +} + +WOLFSSL_LOCAL void poly1305_final_avx(Poly1305* ctx, byte* mac) +{ + word64 rdi, rbx, rax, rsi = 0, rdx = 0, rcx = 0, r11 = 0, r12 = 0, r8 = 0, + r9 = 0, r10 = 0; + word64 zf1; + word64 zf2; + unsigned char cf; + + rdi = (word64)(size_t)ctx; + rbx = (word64)(size_t)mac; + + rax = (word64)(WC_L64(rdi, 608)); + if (((rax & rax)) == (0)) { + goto L_poly1305_avx_final_no_more; + } + WC_S8(rdi, rax + 480) = (byte)(1); + goto L_poly1305_avx_final_cmp_rem; +L_poly1305_avx_final_zero_rem: + WC_S8(rdi, rax + 480) = (byte)(0); +L_poly1305_avx_final_cmp_rem: + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + if ((sword64)(rax) < (sword64)(0x10)) { + goto L_poly1305_avx_final_zero_rem; + } + WC_S8(rdi, 616) = (byte)(0); + rsi = (word64)(rdi + 480); + (void)poly1305_block_avx((Poly1305*)(size_t)rdi, (const unsigned char*)( + size_t)rsi); +L_poly1305_avx_final_no_more: + rax = (word64)(WC_L64(rdi, 24)); + rdx = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + r11 = (word64)(WC_L64(rdi, 48)); + r12 = (word64)(WC_L64(rdi, 56)); + /* h %= p */ + /* h = (h + pad) */ + /* mod 2^130 - 5 */ + r8 = (word64)(rcx); + rcx = (word64)(rcx & 3); + r8 = (word64)(r8 >> 2); + /* Multiply by 5 */ + r8 = (word64)(r8 + r8 * 4); + cf = _addcarry_u64(0, rax, r8, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* Fixup when between (1 << 130) - 1 and (1 << 130) - 5 */ + r8 = (word64)(rax); + r9 = (word64)(rdx); + r10 = (word64)(rcx); + cf = _addcarry_u64(0, r8, 5, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + zf1 = r10; + zf2 = 4; + rax = (r10) == (4) ? r8 : rax; + rdx = (zf1) == (zf2) ? r9 : rdx; + /* h += pad */ + cf = _addcarry_u64(0, rax, r11, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, r12, (unsigned long long*)&rdx); + WC_S64(rbx, 0) = (word64)(rax); + WC_S64(rbx, 8) = (word64)(rdx); + /* Zero out r */ + WC_S64(rdi, 0) = (word64)(0); + WC_S64(rdi, 8) = (word64)(0); + /* Zero out h */ + WC_S64(rdi, 24) = (word64)(0); + WC_S64(rdi, 32) = (word64)(0); + WC_S64(rdi, 40) = (word64)(0); + /* Zero out pad */ + WC_S64(rdi, 48) = (word64)(0); + WC_S64(rdi, 56) = (word64)(0); +} + +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +WOLFSSL_LOCAL void poly1305_calc_powers_avx2(Poly1305* ctx) +{ + word64 rdi, rcx, r8, r9, rax, rdx, rsi, rbx, rbp, r13, r11, r12, r10, r15, + r14; + unsigned char cf; + + rdi = (word64)(size_t)ctx; + + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(0); + /* Convert to 26 bits in 32 */ + rax = (word64)(rcx); + rdx = (word64)(rcx); + rsi = (word64)(rcx); + rbx = (word64)(r8); + rbp = (word64)(r8); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r8 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r9 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 224) = (word32)((word32)rax); + WC_S32(rdi, 228) = (word32)((word32)rdx); + WC_S32(rdi, 232) = (word32)((word32)rsi); + WC_S32(rdi, 236) = (word32)((word32)rbx); + WC_S32(rdi, 240) = (word32)((word32)rbp); + WC_S32(rdi, 244) = (word32)(0); + /* Square 128-bit */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, rax); + r10 = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r15, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Reduce 256-bit to 130-bit */ + rax = (word64)(r12); + rdx = (word64)(r13); + rax = (word64)(rax & -4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)(rdx >> 2); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + rax = (word64)(r12); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* Convert to 26 bits in 32 */ + rax = (word64)(r10); + rdx = (word64)(r10); + rsi = (word64)(r10); + rbx = (word64)(r11); + rbp = (word64)(r11); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r11 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r12 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 256) = (word32)((word32)rax); + WC_S32(rdi, 260) = (word32)((word32)rdx); + WC_S32(rdi, 264) = (word32)((word32)rsi); + WC_S32(rdi, 268) = (word32)((word32)rbx); + WC_S32(rdi, 272) = (word32)((word32)rbp); + WC_S32(rdi, 276) = (word32)(0); + /* Multiply 128-bit by 130-bit */ + /* r1[0] * r2[0] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r10); + r13 = (word64)(rax); + r14 = (word64)(rdx); + /* r1[0] * r2[1] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r11); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* r1[1] * r2[0] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r10); + rsi = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + /* r1[0] * r2[2] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + /* r1[1] * r2[1] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r11); + rbx = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[2] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r15); + rdx = (word64)(rsi); + rbx = (word64)(rbx); + rax = (word64)(rax & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (rbx << 62)); + rbx = (word64)(rbx >> 2); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Convert to 26 bits in 32 */ + rax = (word64)(r13); + rdx = (word64)(r13); + rsi = (word64)(r13); + rbx = (word64)(r14); + rbp = (word64)(r14); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r14 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r15 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 288) = (word32)((word32)rax); + WC_S32(rdi, 292) = (word32)((word32)rdx); + WC_S32(rdi, 296) = (word32)((word32)rsi); + WC_S32(rdi, 300) = (word32)((word32)rbx); + WC_S32(rdi, 304) = (word32)((word32)rbp); + WC_S32(rdi, 308) = (word32)(0); + /* Square 130-bit */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r10); + r13 = (word64)(0); + r8 = (word64)(rax); + r9 = (word64)(rdx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, r15, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rax); + r14 = (word64)(rax); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r10); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r11); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r9); + rdx = (word64)(r13); + r15 = (word64)(r14); + rax = (word64)(rax & -4); + r9 = (word64)(r9 & 3); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r15, (unsigned long long*)&r9); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (r15 << 62)); + r15 = (word64)(r15 >> 2); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r15, (unsigned long long*)&r9); + rax = (word64)(r9); + r9 = (word64)(r9 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* Convert to 26 bits in 32 */ + rax = (word64)(rcx); + rdx = (word64)(rcx); + rsi = (word64)(rcx); + rbx = (word64)(r8); + rbp = (word64)(r8); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r8 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r9 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 320) = (word32)((word32)rax); + WC_S32(rdi, 324) = (word32)((word32)rdx); + WC_S32(rdi, 328) = (word32)((word32)rsi); + WC_S32(rdi, 332) = (word32)((word32)rbx); + WC_S32(rdi, 336) = (word32)((word32)rbp); + WC_S32(rdi, 340) = (word32)(0); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void poly1305_setkey_avx2(Poly1305* ctx, const byte* key) +{ + word64 rdi, rsi; + __m256i y0; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)key; + + (void)poly1305_setkey_avx((Poly1305*)(size_t)rdi, (const byte*)(size_t)rsi); + y0 = _mm256_setzero_si256(); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y0); + WC_S64(rdi, 608) = (word64)(0); + WC_S16(rdi, 616) = (word16)(0); +} + +XALIGNED(32) static const word64 L_poly1305_avx2_blocks_mask[] + WC_X64I_UNUSED = { + 0x0000000003ffffffULL, 0x0000000003ffffffULL, + 0x0000000003ffffffULL, 0x0000000003ffffffULL, +}; + +XALIGNED(32) static const word64 L_poly1305_avx2_blocks_hibit[] + WC_X64I_UNUSED = { + 0x0000000001000000ULL, 0x0000000001000000ULL, + 0x0000000001000000ULL, 0x0000000001000000ULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void poly1305_blocks_avx2(Poly1305* ctx, const unsigned char* m, + size_t bytes) +{ + word64 rdi, rsi, rdx, rsp, r13, r14, rcx, rbx, rax, r8 = 0, r9 = 0, + r10 = 0, r11 = 0, r12 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15; + XALIGNED(32) WC_X64I_SLOT stk[40]; + unsigned char cf; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rdx = (word64)(word64)bytes; + + rsp = (word64)(size_t)stk + 320; + rsp = (word64)(rsp - 320); + r13 = (word64)((word64)(size_t)L_poly1305_avx2_blocks_mask); + r14 = (word64)((word64)(size_t)L_poly1305_avx2_blocks_hibit); + rcx = (word64)(rsp); + rcx = (word64)(rcx & -32); + rcx = (word64)(rcx + 0x20); + y15 = _mm256_setzero_si256(); + rbx = (word64)(rcx); + rax = (word64)(rdi + 64); + rbx = (word64)(rbx + 0xa0); + if ((WC_L16(rdi, 616)) != (0)) { + goto L_poly1305_avx2_blocks_begin_h; + } + /* Load the message data */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_permute2x128_si256(y0, y1, 0x20); + y0 = _mm256_permute2x128_si256(y0, y1, 0x31); + y1 = _mm256_unpacklo_epi32(y2, y0); + y3 = _mm256_unpackhi_epi32(y2, y0); + y0 = _mm256_unpacklo_epi32(y1, y15); + y1 = _mm256_unpackhi_epi32(y1, y15); + y2 = _mm256_unpacklo_epi32(y3, y15); + y3 = _mm256_unpackhi_epi32(y3, y15); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 0)); + y1 = _mm256_slli_epi64(y1, 6); + y2 = _mm256_slli_epi64(y2, 12); + y3 = _mm256_slli_epi64(y3, 18); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + /* Reduce, in place, the message data */ + y10 = _mm256_srli_epi64(y0, 26); + y11 = _mm256_srli_epi64(y3, 26); + y0 = _mm256_and_si256(y0, y14); + y3 = _mm256_and_si256(y3, y14); + y1 = _mm256_add_epi64(y10, y1); + y4 = _mm256_add_epi64(y11, y4); + y10 = _mm256_srli_epi64(y1, 26); + y11 = _mm256_srli_epi64(y4, 26); + y1 = _mm256_and_si256(y1, y14); + y4 = _mm256_and_si256(y4, y14); + y2 = _mm256_add_epi64(y10, y2); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y2, 26); + y0 = _mm256_add_epi64(y12, y0); + y11 = _mm256_srli_epi64(y0, 26); + y2 = _mm256_and_si256(y2, y14); + y0 = _mm256_and_si256(y0, y14); + y3 = _mm256_add_epi64(y10, y3); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y3, 26); + y3 = _mm256_and_si256(y3, y14); + y4 = _mm256_add_epi64(y10, y4); + rsi = (word64)(rsi + 0x40); + rdx = (word64)(rdx - 0x40); + if ((rdx) == (0)) { + goto L_poly1305_avx2_blocks_store; + } + goto L_poly1305_avx2_blocks_load_r4; +L_poly1305_avx2_blocks_begin_h: + /* Load the H values. */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 128)); + /* Check if there is a power of r to load - otherwise use r^4. */ + if ((WC_L8(rdi, 616)) == (0)) { + goto L_poly1305_avx2_blocks_load_r4; + } + /* Load the 4 powers of r - r^4, r^3, r^2, r^1. */ + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y5 = _mm256_permute4x64_epi64(y5, 0xd8); + y6 = _mm256_permute4x64_epi64(y6, 0xd8); + y7 = _mm256_permute4x64_epi64(y7, 0xd8); + y8 = _mm256_permute4x64_epi64(y8, 0xd8); + y10 = _mm256_unpacklo_epi64(y5, y6); + y11 = _mm256_unpackhi_epi64(y5, y6); + y12 = _mm256_unpacklo_epi64(y7, y8); + y13 = _mm256_unpackhi_epi64(y7, y8); + y5 = _mm256_permute2x128_si256(y10, y12, 0x20); + y7 = _mm256_permute2x128_si256(y10, y12, 0x31); + y9 = _mm256_permute2x128_si256(y11, y13, 0x20); + y6 = _mm256_srli_epi64(y5, 32); + y8 = _mm256_srli_epi64(y7, 32); + goto L_poly1305_avx2_blocks_mul_5; +L_poly1305_avx2_blocks_load_r4: + /* Load r^4 into all four positions. */ + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y5 = _mm256_permute4x64_epi64(y13, 0); + y14 = _mm256_srli_epi64(y13, 32); + y7 = _mm256_permute4x64_epi64(y13, 0x55); + y9 = _mm256_permute4x64_epi64(y13, 0xaa); + y6 = _mm256_permute4x64_epi64(y14, 0); + y8 = _mm256_permute4x64_epi64(y14, 0x55); +L_poly1305_avx2_blocks_mul_5: + /* Multiply top 4 26-bit values of all four H by 5 */ + y10 = _mm256_slli_epi32(y6, 2); + y11 = _mm256_slli_epi32(y7, 2); + y12 = _mm256_slli_epi32(y8, 2); + y13 = _mm256_slli_epi32(y9, 2); + y10 = _mm256_add_epi64(y6, y10); + y11 = _mm256_add_epi64(y7, y11); + y12 = _mm256_add_epi64(y8, y12); + y13 = _mm256_add_epi64(y9, y13); + /* Store powers of r and multiple of 5 for use in multiply. */ + _mm256_storeu_si256((__m256i*)WC_PW(rbx, 0), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rbx, 32), y11); + _mm256_storeu_si256((__m256i*)WC_PW(rbx, 64), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rbx, 96), y13); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y9); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, 0)); + /* If not finished then loop over data */ + if ((WC_L8(rdi, 616)) != (1)) { + goto L_poly1305_avx2_blocks_start; + } + /* Do last multiply, reduce, add the four H together and move to */ + /* 32-bit registers */ + y5 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y8 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y9 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + y5 = _mm256_bsrli_epi128(y0, 8); + y6 = _mm256_bsrli_epi128(y1, 8); + y7 = _mm256_bsrli_epi128(y2, 8); + y8 = _mm256_bsrli_epi128(y3, 8); + y9 = _mm256_bsrli_epi128(y4, 8); + y0 = _mm256_add_epi64(y5, y0); + y1 = _mm256_add_epi64(y6, y1); + y2 = _mm256_add_epi64(y7, y2); + y3 = _mm256_add_epi64(y8, y3); + y4 = _mm256_add_epi64(y9, y4); + y5 = _mm256_permute4x64_epi64(y0, 2); + y6 = _mm256_permute4x64_epi64(y1, 2); + y7 = _mm256_permute4x64_epi64(y2, 2); + y8 = _mm256_permute4x64_epi64(y3, 2); + y9 = _mm256_permute4x64_epi64(y4, 2); + y0 = _mm256_add_epi64(y5, y0); + y1 = _mm256_add_epi64(y6, y1); + y2 = _mm256_add_epi64(y7, y2); + y3 = _mm256_add_epi64(y8, y3); + y4 = _mm256_add_epi64(y9, y4); + r8 = (word32)((word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y0))); + r9 = (word32)((word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y1))); + r10 = (word32)((word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y2))); + r11 = (word32)((word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y3))); + r12 = (word32)((word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y4))); + goto L_poly1305_avx2_blocks_end_calc; +L_poly1305_avx2_blocks_start: + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y7 = _mm256_permute2x128_si256(y5, y6, 0x20); + y5 = _mm256_permute2x128_si256(y5, y6, 0x31); + y6 = _mm256_unpacklo_epi32(y7, y5); + y8 = _mm256_unpackhi_epi32(y7, y5); + y5 = _mm256_unpacklo_epi32(y6, y15); + y6 = _mm256_unpackhi_epi32(y6, y15); + y7 = _mm256_unpacklo_epi32(y8, y15); + y8 = _mm256_unpackhi_epi32(y8, y15); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 0)); + y6 = _mm256_slli_epi64(y6, 6); + y7 = _mm256_slli_epi64(y7, 12); + y8 = _mm256_slli_epi64(y8, 18); + y10 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 0))); + y5 = _mm256_add_epi64(y10, y5); + y10 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y11 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 32))); + y6 = _mm256_add_epi64(y11, y6); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y12 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y7 = _mm256_add_epi64(y12, y7); + y5 = _mm256_add_epi64(y10, y5); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 64))); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y8 = _mm256_add_epi64(y13, y8); + y6 = _mm256_add_epi64(y11, y6); + y13 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y10 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbx, + 96))); + y12 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y9 = _mm256_add_epi64(y13, y9); + y6 = _mm256_add_epi64(y10, y6); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y7 = _mm256_add_epi64(y11, y7); + y10 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y8 = _mm256_add_epi64(y12, y8); + y11 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y5 = _mm256_add_epi64(y13, y5); + y13 = _mm256_mul_epu32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y9 = _mm256_add_epi64(y13, y9); + y13 = _mm256_mul_epu32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y6 = _mm256_add_epi64(y10, y6); + y10 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y7 = _mm256_add_epi64(y11, y7); + y11 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y8 = _mm256_add_epi64(y12, y8); + y12 = _mm256_mul_epu32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y9 = _mm256_add_epi64(y13, y9); + y7 = _mm256_add_epi64(y10, y7); + y13 = _mm256_mul_epu32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y8 = _mm256_add_epi64(y11, y8); + y9 = _mm256_add_epi64(y12, y9); + y9 = _mm256_add_epi64(y13, y9); + y10 = _mm256_srli_epi64(y5, 26); + y11 = _mm256_srli_epi64(y8, 26); + y5 = _mm256_and_si256(y5, y14); + y8 = _mm256_and_si256(y8, y14); + y6 = _mm256_add_epi64(y10, y6); + y9 = _mm256_add_epi64(y11, y9); + y10 = _mm256_srli_epi64(y6, 26); + y11 = _mm256_srli_epi64(y9, 26); + y1 = _mm256_and_si256(y6, y14); + y4 = _mm256_and_si256(y9, y14); + y7 = _mm256_add_epi64(y10, y7); + y12 = _mm256_slli_epi32(y11, 2); + y12 = _mm256_add_epi32(y11, y12); + y10 = _mm256_srli_epi64(y7, 26); + y5 = _mm256_add_epi64(y12, y5); + y11 = _mm256_srli_epi64(y5, 26); + y2 = _mm256_and_si256(y7, y14); + y0 = _mm256_and_si256(y5, y14); + y8 = _mm256_add_epi64(y10, y8); + y1 = _mm256_add_epi64(y11, y1); + y10 = _mm256_srli_epi64(y8, 26); + y3 = _mm256_and_si256(y8, y14); + y4 = _mm256_add_epi64(y10, y4); + rsi = (word64)(rsi + 0x40); + rdx = (word64)(rdx - 0x40); + if ((rdx) != (0)) { + goto L_poly1305_avx2_blocks_start; + } +L_poly1305_avx2_blocks_store: + /* Store four H values - state */ + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); +L_poly1305_avx2_blocks_end_calc: + if ((WC_L8(rdi, 616)) == (0)) { + goto L_poly1305_avx2_blocks_complete; + } + rax = (word64)(r8); + rdx = (word64)(r10); + rcx = (word64)(r12); + rdx = (word64)(rdx >> 12); + rcx = (word64)(rcx >> 24); + r9 = (word64)(r9 << 26); + r10 = (word64)(r10 << 52); + r11 = (word64)(r11 << 14); + r12 = (word64)(r12 << 40); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rax, r10, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rdx, r12, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(rcx); + rcx = (word64)(rcx & 3); + r8 = (word64)(r8 >> 2); + r8 = (word64)(r8 + r8 * 4); + cf = _addcarry_u64(0, rax, r8, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 24) = (word64)(rax); + WC_S64(rdi, 32) = (word64)(rdx); + WC_S64(rdi, 40) = (word64)(rcx); +L_poly1305_avx2_blocks_complete: + WC_S8(rdi, 617) = (byte)(1); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void poly1305_final_avx2(Poly1305* ctx, byte* mac) +{ + word64 rdi, rsi, rcx = 0, rsp, rdx = 0, rax = 0, r8 = 0; + __m256i y0 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)mac; + + rsp = (word64)(size_t)stk + 32; + WC_S8(rdi, 616) = (byte)(1); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, 617)) & 0xff); + if (((byte)rcx) == (0)) { + goto L_poly1305_avx2_final_done_blocks_X4; + } + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rsi; + rdx = (word64)(0x40); + rsi = (word64)(0); + (void)poly1305_blocks_avx2((Poly1305*)(size_t)rdi, (const unsigned char*)( + size_t)rsi, (size_t)rdx); + rsi = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); +L_poly1305_avx2_final_done_blocks_X4: + rax = (word64)(WC_L64(rdi, 608)); + rcx = (word64)(rax); + rcx = (word64)(rcx & -16); + if (((byte)rcx) == (0)) { + goto L_poly1305_avx2_final_done_blocks; + } + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rcx; + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rax; + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rsi; + rdx = (word64)(rcx); + rsi = (word64)(rdi + 480); + (void)poly1305_blocks_avx((Poly1305*)(size_t)rdi, (const unsigned char*)( + size_t)rsi, (size_t)rdx); + rsi = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); + rax = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); + rcx = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); +L_poly1305_avx2_final_done_blocks: + WC_S64(rdi, 608) = (word64)(WC_L64(rdi, 608) - rcx); + rdx = (word64)(0); + goto L_poly1305_avx2_final_cmp_copy; +L_poly1305_avx2_final_start_copy: + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, rcx + 480)) & 0xff); + WC_S8(rdi, rdx + 480) = (byte)((byte)r8); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx + 1) & 0xff); + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)((byte)rdx + 1) & 0xff); +L_poly1305_avx2_final_cmp_copy: + if (((byte)rax) != ((byte)rcx)) { + goto L_poly1305_avx2_final_start_copy; + } + (void)poly1305_final_avx((Poly1305*)(size_t)rdi, (byte*)(size_t)rsi); + y0 = _mm256_setzero_si256(); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y0); + WC_S64(rdi, 608) = (word64)(0); + WC_S16(rdi, 616) = (word16)(0); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX512 +WOLFSSL_LOCAL void poly1305_calc_powers_avx512(Poly1305* ctx) +{ + word64 rdi, rsp, rcx, r8, r9, rax, rdx, rsi, rbx, rbp, r13, r11, r12, r10, + r15, r14; + XALIGNED(32) WC_X64I_SLOT stk[12]; + unsigned char cf; + + rdi = (word64)(size_t)ctx; + + rsp = (word64)(size_t)stk + 96; + rsp = (word64)(rsp - 96); + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(0); + /* Convert to 26 bits in 32 */ + rax = (word64)(rcx); + rdx = (word64)(rcx); + rsi = (word64)(rcx); + rbx = (word64)(r8); + rbp = (word64)(r8); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r8 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r9 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 224) = (word32)((word32)rax); + WC_S32(rdi, 228) = (word32)((word32)rdx); + WC_S32(rdi, 232) = (word32)((word32)rsi); + WC_S32(rdi, 236) = (word32)((word32)rbx); + WC_S32(rdi, 240) = (word32)((word32)rbp); + WC_S32(rdi, 244) = (word32)(0); + WC_S64(rsp, 0) = (word64)(rcx); + WC_S64(rsp, 8) = (word64)(r8); + WC_S64(rsp, 16) = (word64)(r9); + /* Square 128-bit */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, rax); + r10 = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r15, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Reduce 256-bit to 130-bit */ + rax = (word64)(r12); + rdx = (word64)(r13); + rax = (word64)(rax & -4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)(rdx >> 2); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + rax = (word64)(r12); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* Convert to 26 bits in 32 */ + rax = (word64)(r10); + rdx = (word64)(r10); + rsi = (word64)(r10); + rbx = (word64)(r11); + rbp = (word64)(r11); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r11 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r12 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 256) = (word32)((word32)rax); + WC_S32(rdi, 260) = (word32)((word32)rdx); + WC_S32(rdi, 264) = (word32)((word32)rsi); + WC_S32(rdi, 268) = (word32)((word32)rbx); + WC_S32(rdi, 272) = (word32)((word32)rbp); + WC_S32(rdi, 276) = (word32)(0); + WC_S64(rsp, 24) = (word64)(r10); + WC_S64(rsp, 32) = (word64)(r11); + WC_S64(rsp, 40) = (word64)(r12); + /* Multiply 128-bit by 130-bit */ + /* r1[0] * r2[0] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r10); + r13 = (word64)(rax); + r14 = (word64)(rdx); + /* r1[0] * r2[1] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r11); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* r1[1] * r2[0] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r10); + rsi = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + /* r1[0] * r2[2] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + /* r1[1] * r2[1] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r11); + rbx = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[2] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r15); + rdx = (word64)(rsi); + rbx = (word64)(rbx); + rax = (word64)(rax & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (rbx << 62)); + rbx = (word64)(rbx >> 2); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Convert to 26 bits in 32 */ + rax = (word64)(r13); + rdx = (word64)(r13); + rsi = (word64)(r13); + rbx = (word64)(r14); + rbp = (word64)(r14); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r14 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r15 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 288) = (word32)((word32)rax); + WC_S32(rdi, 292) = (word32)((word32)rdx); + WC_S32(rdi, 296) = (word32)((word32)rsi); + WC_S32(rdi, 300) = (word32)((word32)rbx); + WC_S32(rdi, 304) = (word32)((word32)rbp); + WC_S32(rdi, 308) = (word32)(0); + WC_S64(rsp, 48) = (word64)(r13); + WC_S64(rsp, 56) = (word64)(r14); + WC_S64(rsp, 64) = (word64)(r15); + /* Square 130-bit */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r10); + r13 = (word64)(0); + r8 = (word64)(rax); + r9 = (word64)(rdx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, r15, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rax); + r14 = (word64)(rax); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r10); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r11); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r9); + rdx = (word64)(r13); + r15 = (word64)(r14); + rax = (word64)(rax & -4); + r9 = (word64)(r9 & 3); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r15, (unsigned long long*)&r9); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (r15 << 62)); + r15 = (word64)(r15 >> 2); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r15, (unsigned long long*)&r9); + rax = (word64)(r9); + r9 = (word64)(r9 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* Convert to 26 bits in 32 */ + rax = (word64)(rcx); + rdx = (word64)(rcx); + rsi = (word64)(rcx); + rbx = (word64)(r8); + rbp = (word64)(r8); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r8 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r9 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 320) = (word32)((word32)rax); + WC_S32(rdi, 324) = (word32)((word32)rdx); + WC_S32(rdi, 328) = (word32)((word32)rsi); + WC_S32(rdi, 332) = (word32)((word32)rbx); + WC_S32(rdi, 336) = (word32)((word32)rbp); + WC_S32(rdi, 340) = (word32)(0); + WC_S64(rsp, 72) = (word64)(rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r10 = (word64)(WC_L64(rsp, 0)); + r11 = (word64)(WC_L64(rsp, 8)); + r12 = (word64)(WC_L64(rsp, 16)); + /* Multiply 128-bit by 130-bit */ + /* r1[0] * r2[0] */ + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(rax); + r14 = (word64)(rdx); + /* r1[0] * r2[1] */ + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, r8); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* r1[1] * r2[0] */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, rcx); + rsi = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + /* r1[0] * r2[2] */ + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, r9); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + /* r1[1] * r2[1] */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r8); + rbx = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[2] */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r9); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r15); + rdx = (word64)(rsi); + rbx = (word64)(rbx); + rax = (word64)(rax & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (rbx << 62)); + rbx = (word64)(rbx >> 2); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Convert to 26 bits in 32 */ + rax = (word64)(r13); + rdx = (word64)(r13); + rsi = (word64)(r13); + rbx = (word64)(r14); + rbp = (word64)(r14); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r14 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r15 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 624) = (word32)((word32)rax); + WC_S32(rdi, 628) = (word32)((word32)rdx); + WC_S32(rdi, 632) = (word32)((word32)rsi); + WC_S32(rdi, 636) = (word32)((word32)rbx); + WC_S32(rdi, 640) = (word32)((word32)rbp); + WC_S32(rdi, 644) = (word32)(0); + /* Square 130-bit */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, rax); + r10 = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r15, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rax); + r14 = (word64)(rax); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, r8); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r12); + rdx = (word64)(r13); + r15 = (word64)(r14); + rax = (word64)(rax & -4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (r15 << 62)); + r15 = (word64)(r15 >> 2); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)(r12); + r12 = (word64)(r12 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* Convert to 26 bits in 32 */ + rax = (word64)(r10); + rdx = (word64)(r10); + rsi = (word64)(r10); + rbx = (word64)(r11); + rbp = (word64)(r11); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r11 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r12 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 720) = (word32)((word32)rax); + WC_S32(rdi, 724) = (word32)((word32)rdx); + WC_S32(rdi, 728) = (word32)((word32)rsi); + WC_S32(rdi, 732) = (word32)((word32)rbx); + WC_S32(rdi, 736) = (word32)((word32)rbp); + WC_S32(rdi, 740) = (word32)(0); + rcx = (word64)(WC_L64(rsp, 48)); + r8 = (word64)(WC_L64(rsp, 56)); + r9 = (word64)(WC_L64(rsp, 64)); + /* Square 130-bit */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, rax); + r10 = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r15, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rax); + r14 = (word64)(rax); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, r8); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r12); + rdx = (word64)(r13); + r15 = (word64)(r14); + rax = (word64)(rax & -4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (r15 << 62)); + r15 = (word64)(r15 >> 2); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)(r12); + r12 = (word64)(r12 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* Convert to 26 bits in 32 */ + rax = (word64)(r10); + rdx = (word64)(r10); + rsi = (word64)(r10); + rbx = (word64)(r11); + rbp = (word64)(r11); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r11 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r12 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 656) = (word32)((word32)rax); + WC_S32(rdi, 660) = (word32)((word32)rdx); + WC_S32(rdi, 664) = (word32)((word32)rsi); + WC_S32(rdi, 668) = (word32)((word32)rbx); + WC_S32(rdi, 672) = (word32)((word32)rbp); + WC_S32(rdi, 676) = (word32)(0); + rcx = (word64)(WC_L64(rsp, 0)); + r8 = (word64)(WC_L64(rsp, 8)); + r9 = (word64)(WC_L64(rsp, 16)); + /* Multiply 128-bit by 130-bit */ + /* r1[0] * r2[0] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r10); + r13 = (word64)(rax); + r14 = (word64)(rdx); + /* r1[0] * r2[1] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r11); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* r1[1] * r2[0] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r10); + rsi = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + /* r1[0] * r2[2] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + /* r1[1] * r2[1] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r11); + rbx = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[2] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r15); + rdx = (word64)(rsi); + rbx = (word64)(rbx); + rax = (word64)(rax & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (rbx << 62)); + rbx = (word64)(rbx >> 2); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Convert to 26 bits in 32 */ + rax = (word64)(r13); + rdx = (word64)(r13); + rsi = (word64)(r13); + rbx = (word64)(r14); + rbp = (word64)(r14); + rdx = (word64)(rdx >> 26); + rsi = (word64)((rsi >> 52) | (r14 << 12)); + rbx = (word64)(rbx >> 14); + rbp = (word64)((rbp >> 40) | (r15 << 24)); + rax = (word64)(rax & 0x3ffffff); + rdx = (word64)(rdx & 0x3ffffff); + rsi = (word64)(rsi & 0x3ffffff); + rbx = (word64)(rbx & 0x3ffffff); + rbp = (word64)(rbp & 0x3ffffff); + WC_S32(rdi, 688) = (word32)((word32)rax); + WC_S32(rdi, 692) = (word32)((word32)rdx); + WC_S32(rdi, 696) = (word32)((word32)rsi); + WC_S32(rdi, 700) = (word32)((word32)rbx); + WC_S32(rdi, 704) = (word32)((word32)rbp); + WC_S32(rdi, 708) = (word32)(0); +} + +XALIGNED(32) static const word64 L_poly1305_avx512_blocks_mask[] + WC_X64I_UNUSED = { + 0x0000000003ffffffULL, 0x0000000003ffffffULL, + 0x0000000003ffffffULL, 0x0000000003ffffffULL, + 0x0000000003ffffffULL, 0x0000000003ffffffULL, + 0x0000000003ffffffULL, 0x0000000003ffffffULL, +}; + +XALIGNED(32) static const word64 L_poly1305_avx512_blocks_hibit[] + WC_X64I_UNUSED = { + 0x0000000001000000ULL, 0x0000000001000000ULL, + 0x0000000001000000ULL, 0x0000000001000000ULL, + 0x0000000001000000ULL, 0x0000000001000000ULL, + 0x0000000001000000ULL, 0x0000000001000000ULL, +}; + +XALIGNED(32) static const word64 L_poly1305_avx512_blocks_permidx[] + WC_X64I_UNUSED = { + 0x0000000400000000ULL, 0x0000000c00000008ULL, + 0x0000001400000010ULL, 0x0000001c00000018ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000001ULL, 0x0000000d00000009ULL, + 0x0000001500000011ULL, 0x0000001d00000019ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000600000002ULL, 0x0000000e0000000aULL, + 0x0000001600000012ULL, 0x0000001e0000001aULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000700000003ULL, 0x0000000f0000000bULL, + 0x0000001700000013ULL, 0x0000001f0000001bULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_poly1305_avx512_blocks_rxidx[] + WC_X64I_UNUSED = { + 0x0000001000000018ULL, 0x0000000000000008ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000001100000019ULL, 0x0000000100000009ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x000000120000001aULL, 0x000000020000000aULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x000000130000001bULL, 0x000000030000000bULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x000000140000001cULL, 0x000000040000000cULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void poly1305_blocks_avx512(Poly1305* ctx, const unsigned char* m, + size_t bytes) +{ + word64 rdi, rsi, rdx, rcx, r13, r14, r15, rax, r8 = 0, r9 = 0, r10 = 0, + r11 = 0, r12 = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), + z8 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), + z12 = _mm512_setzero_si512(), z13 = _mm512_setzero_si512(), + z14 = _mm512_setzero_si512(), z15 = _mm512_setzero_si512(), + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(), + z22 = _mm512_setzero_si512(), z23 = _mm512_setzero_si512(), z24, + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(), + z27 = _mm512_setzero_si512(); + unsigned char cf; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rdx = (word64)(word64)bytes; + + rcx = (word64)((word64)(size_t)L_poly1305_avx512_blocks_mask); + r13 = (word64)((word64)(size_t)L_poly1305_avx512_blocks_hibit); + r14 = (word64)((word64)(size_t)L_poly1305_avx512_blocks_permidx); + r15 = (word64)((word64)(size_t)L_poly1305_avx512_blocks_rxidx); + z24 = _mm512_setzero_si512(); + rax = (word64)(rdi + 64); + if ((WC_L16(rdi, 616)) != (0)) { + goto L_poly1305_avx512_blocks_begin_h; + } + /* Load the message data */ + z25 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z27 = _mm512_loadu_si512((const void*)WC_PR(r14, 0)); + z27 = _mm512_permutex2var_epi32(z25, z27, z26); + z0 = _mm512_cvtepu32_epi64(_mm512_castsi512_si256(z27)); + z27 = _mm512_loadu_si512((const void*)WC_PR(r14, 64)); + z27 = _mm512_permutex2var_epi32(z25, z27, z26); + z1 = _mm512_cvtepu32_epi64(_mm512_castsi512_si256(z27)); + z27 = _mm512_loadu_si512((const void*)WC_PR(r14, 128)); + z27 = _mm512_permutex2var_epi32(z25, z27, z26); + z2 = _mm512_cvtepu32_epi64(_mm512_castsi512_si256(z27)); + z27 = _mm512_loadu_si512((const void*)WC_PR(r14, 192)); + z27 = _mm512_permutex2var_epi32(z25, z27, z26); + z3 = _mm512_cvtepu32_epi64(_mm512_castsi512_si256(z27)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r13, 0)); + z1 = _mm512_slli_epi64(z1, 6); + z2 = _mm512_slli_epi64(z2, 12); + z3 = _mm512_slli_epi64(z3, 18); + z23 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + /* Reduce, in place, the message data */ + z19 = _mm512_srli_epi64(z0, 26); + z20 = _mm512_srli_epi64(z3, 26); + z0 = _mm512_and_si512(z0, z23); + z3 = _mm512_and_si512(z3, z23); + z1 = _mm512_add_epi64(z19, z1); + z4 = _mm512_add_epi64(z20, z4); + z19 = _mm512_srli_epi64(z1, 26); + z20 = _mm512_srli_epi64(z4, 26); + z1 = _mm512_and_si512(z1, z23); + z4 = _mm512_and_si512(z4, z23); + z2 = _mm512_add_epi64(z19, z2); + z21 = _mm512_slli_epi32(z20, 2); + z21 = _mm512_add_epi32(z20, z21); + z19 = _mm512_srli_epi64(z2, 26); + z0 = _mm512_add_epi64(z21, z0); + z20 = _mm512_srli_epi64(z0, 26); + z2 = _mm512_and_si512(z2, z23); + z0 = _mm512_and_si512(z0, z23); + z3 = _mm512_add_epi64(z19, z3); + z1 = _mm512_add_epi64(z20, z1); + z19 = _mm512_srli_epi64(z3, 26); + z3 = _mm512_and_si512(z3, z23); + z4 = _mm512_add_epi64(z19, z4); + rsi = (word64)(rsi + 0x80); + rdx = (word64)(rdx - 0x80); + if ((rdx) == (0)) { + goto L_poly1305_avx512_blocks_store; + } + goto L_poly1305_avx512_blocks_load_r8; +L_poly1305_avx512_blocks_begin_h: + /* Load the H values. */ + z0 = _mm512_cvtepu32_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + z1 = _mm512_cvtepu32_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + z2 = _mm512_cvtepu32_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + z3 = _mm512_cvtepu32_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + z4 = _mm512_cvtepu32_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + /* Check if there is a power of r to load - otherwise use r^8. */ + if ((WC_L8(rdi, 616)) == (0)) { + goto L_poly1305_avx512_blocks_load_r8; + } + /* Load the 8 powers of r - r^8 .. r^1. */ + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 224)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rdi, 288)); + z16 = _mm512_loadu_si512((const void*)WC_PR(rdi, 624)); + z17 = _mm512_loadu_si512((const void*)WC_PR(rdi, 688)); + z18 = _mm512_loadu_si512((const void*)WC_PR(r15, 0)); + z18 = _mm512_permutex2var_epi32(z16, z18, z17); + z20 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z18))); + z19 = _mm512_loadu_si512((const void*)WC_PR(r15, 0)); + z19 = _mm512_permutex2var_epi32(z14, z19, z15); + z21 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z19))); + z5 = _mm512_inserti64x4(z20, _mm512_castsi512_si256(z21), 1); + z18 = _mm512_loadu_si512((const void*)WC_PR(r15, 64)); + z18 = _mm512_permutex2var_epi32(z16, z18, z17); + z20 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z18))); + z19 = _mm512_loadu_si512((const void*)WC_PR(r15, 64)); + z19 = _mm512_permutex2var_epi32(z14, z19, z15); + z21 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z19))); + z6 = _mm512_inserti64x4(z20, _mm512_castsi512_si256(z21), 1); + z18 = _mm512_loadu_si512((const void*)WC_PR(r15, 128)); + z18 = _mm512_permutex2var_epi32(z16, z18, z17); + z20 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z18))); + z19 = _mm512_loadu_si512((const void*)WC_PR(r15, 128)); + z19 = _mm512_permutex2var_epi32(z14, z19, z15); + z21 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z19))); + z7 = _mm512_inserti64x4(z20, _mm512_castsi512_si256(z21), 1); + z18 = _mm512_loadu_si512((const void*)WC_PR(r15, 192)); + z18 = _mm512_permutex2var_epi32(z16, z18, z17); + z20 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z18))); + z19 = _mm512_loadu_si512((const void*)WC_PR(r15, 192)); + z19 = _mm512_permutex2var_epi32(z14, z19, z15); + z21 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z19))); + z8 = _mm512_inserti64x4(z20, _mm512_castsi512_si256(z21), 1); + z18 = _mm512_loadu_si512((const void*)WC_PR(r15, 256)); + z18 = _mm512_permutex2var_epi32(z16, z18, z17); + z20 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z18))); + z19 = _mm512_loadu_si512((const void*)WC_PR(r15, 256)); + z19 = _mm512_permutex2var_epi32(z14, z19, z15); + z21 = _mm512_zextsi256_si512(_mm256_cvtepu32_epi64(_mm512_castsi512_si128( + z19))); + z9 = _mm512_inserti64x4(z20, _mm512_castsi512_si256(z21), 1); + goto L_poly1305_avx512_blocks_mul_5; +L_poly1305_avx512_blocks_load_r8: + /* Load r^8 into all eight lanes. */ + z5 = _mm512_set1_epi32((int)WC_L32(rdi, 720)); + z6 = _mm512_set1_epi32((int)WC_L32(rdi, 724)); + z7 = _mm512_set1_epi32((int)WC_L32(rdi, 728)); + z8 = _mm512_set1_epi32((int)WC_L32(rdi, 732)); + z9 = _mm512_set1_epi32((int)WC_L32(rdi, 736)); +L_poly1305_avx512_blocks_mul_5: + /* Multiply top 4 26-bit values of all eight powers by 5 */ + z10 = _mm512_slli_epi32(z6, 2); + z11 = _mm512_slli_epi32(z7, 2); + z12 = _mm512_slli_epi32(z8, 2); + z13 = _mm512_slli_epi32(z9, 2); + z10 = _mm512_add_epi64(z6, z10); + z11 = _mm512_add_epi64(z7, z11); + z12 = _mm512_add_epi64(z8, z12); + z13 = _mm512_add_epi64(z9, z13); + z23 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + /* If not finished then loop over data */ + if ((WC_L8(rdi, 616)) != (1)) { + goto L_poly1305_avx512_blocks_start; + } + /* Do last multiply, reduce, add the eight H together and move to */ + /* 32-bit registers */ + z14 = _mm512_mul_epu32(z4, z10); + z19 = _mm512_mul_epu32(z3, z11); + z15 = _mm512_mul_epu32(z4, z11); + z20 = _mm512_mul_epu32(z3, z12); + z16 = _mm512_mul_epu32(z4, z12); + z14 = _mm512_add_epi64(z19, z14); + z21 = _mm512_mul_epu32(z2, z12); + z17 = _mm512_mul_epu32(z4, z13); + z15 = _mm512_add_epi64(z20, z15); + z22 = _mm512_mul_epu32(z1, z13); + z19 = _mm512_mul_epu32(z2, z13); + z14 = _mm512_add_epi64(z21, z14); + z20 = _mm512_mul_epu32(z3, z13); + z21 = _mm512_mul_epu32(z3, z5); + z14 = _mm512_add_epi64(z22, z14); + z18 = _mm512_mul_epu32(z4, z5); + z15 = _mm512_add_epi64(z19, z15); + z22 = _mm512_mul_epu32(z0, z5); + z16 = _mm512_add_epi64(z20, z16); + z19 = _mm512_mul_epu32(z1, z5); + z17 = _mm512_add_epi64(z21, z17); + z20 = _mm512_mul_epu32(z2, z5); + z21 = _mm512_mul_epu32(z2, z6); + z14 = _mm512_add_epi64(z22, z14); + z22 = _mm512_mul_epu32(z3, z6); + z15 = _mm512_add_epi64(z19, z15); + z19 = _mm512_mul_epu32(z0, z6); + z16 = _mm512_add_epi64(z20, z16); + z20 = _mm512_mul_epu32(z1, z6); + z17 = _mm512_add_epi64(z21, z17); + z21 = _mm512_mul_epu32(z1, z7); + z18 = _mm512_add_epi64(z22, z18); + z22 = _mm512_mul_epu32(z2, z7); + z15 = _mm512_add_epi64(z19, z15); + z19 = _mm512_mul_epu32(z0, z7); + z16 = _mm512_add_epi64(z20, z16); + z20 = _mm512_mul_epu32(z0, z8); + z17 = _mm512_add_epi64(z21, z17); + z21 = _mm512_mul_epu32(z1, z8); + z18 = _mm512_add_epi64(z22, z18); + z16 = _mm512_add_epi64(z19, z16); + z22 = _mm512_mul_epu32(z0, z9); + z17 = _mm512_add_epi64(z20, z17); + z18 = _mm512_add_epi64(z21, z18); + z18 = _mm512_add_epi64(z22, z18); + z19 = _mm512_srli_epi64(z14, 26); + z20 = _mm512_srli_epi64(z17, 26); + z14 = _mm512_and_si512(z14, z23); + z17 = _mm512_and_si512(z17, z23); + z15 = _mm512_add_epi64(z19, z15); + z18 = _mm512_add_epi64(z20, z18); + z19 = _mm512_srli_epi64(z15, 26); + z20 = _mm512_srli_epi64(z18, 26); + z1 = _mm512_and_si512(z15, z23); + z4 = _mm512_and_si512(z18, z23); + z16 = _mm512_add_epi64(z19, z16); + z21 = _mm512_slli_epi32(z20, 2); + z21 = _mm512_add_epi32(z20, z21); + z19 = _mm512_srli_epi64(z16, 26); + z14 = _mm512_add_epi64(z21, z14); + z20 = _mm512_srli_epi64(z14, 26); + z2 = _mm512_and_si512(z16, z23); + z0 = _mm512_and_si512(z14, z23); + z17 = _mm512_add_epi64(z19, z17); + z1 = _mm512_add_epi64(z20, z1); + z19 = _mm512_srli_epi64(z17, 26); + z3 = _mm512_and_si512(z17, z23); + z4 = _mm512_add_epi64(z19, z4); + z14 = _mm512_shuffle_i64x2(z0, z0, 0x4e); + z15 = _mm512_shuffle_i64x2(z1, z1, 0x4e); + z16 = _mm512_shuffle_i64x2(z2, z2, 0x4e); + z17 = _mm512_shuffle_i64x2(z3, z3, 0x4e); + z18 = _mm512_shuffle_i64x2(z4, z4, 0x4e); + z0 = _mm512_add_epi64(z14, z0); + z1 = _mm512_add_epi64(z15, z1); + z2 = _mm512_add_epi64(z16, z2); + z3 = _mm512_add_epi64(z17, z3); + z4 = _mm512_add_epi64(z18, z4); + z14 = _mm512_bsrli_epi128(z0, 8); + z15 = _mm512_bsrli_epi128(z1, 8); + z16 = _mm512_bsrli_epi128(z2, 8); + z17 = _mm512_bsrli_epi128(z3, 8); + z18 = _mm512_bsrli_epi128(z4, 8); + z0 = _mm512_add_epi64(z14, z0); + z1 = _mm512_add_epi64(z15, z1); + z2 = _mm512_add_epi64(z16, z2); + z3 = _mm512_add_epi64(z17, z3); + z4 = _mm512_add_epi64(z18, z4); + z14 = _mm512_permutex_epi64(z0, 2); + z15 = _mm512_permutex_epi64(z1, 2); + z16 = _mm512_permutex_epi64(z2, 2); + z17 = _mm512_permutex_epi64(z3, 2); + z18 = _mm512_permutex_epi64(z4, 2); + z0 = _mm512_add_epi64(z14, z0); + z1 = _mm512_add_epi64(z15, z1); + z2 = _mm512_add_epi64(z16, z2); + z3 = _mm512_add_epi64(z17, z3); + z4 = _mm512_add_epi64(z18, z4); + r8 = (word32)((word32)_mm_cvtsi128_si32(_mm512_castsi512_si128(z0))); + r9 = (word32)((word32)_mm_cvtsi128_si32(_mm512_castsi512_si128(z1))); + r10 = (word32)((word32)_mm_cvtsi128_si32(_mm512_castsi512_si128(z2))); + r11 = (word32)((word32)_mm_cvtsi128_si32(_mm512_castsi512_si128(z3))); + r12 = (word32)((word32)_mm_cvtsi128_si32(_mm512_castsi512_si128(z4))); + goto L_poly1305_avx512_blocks_end_calc; +L_poly1305_avx512_blocks_start: + z25 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z26 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z27 = _mm512_loadu_si512((const void*)WC_PR(r14, 0)); + z27 = _mm512_permutex2var_epi32(z25, z27, z26); + z14 = _mm512_cvtepu32_epi64(_mm512_castsi512_si256(z27)); + z27 = _mm512_loadu_si512((const void*)WC_PR(r14, 64)); + z27 = _mm512_permutex2var_epi32(z25, z27, z26); + z15 = _mm512_cvtepu32_epi64(_mm512_castsi512_si256(z27)); + z27 = _mm512_loadu_si512((const void*)WC_PR(r14, 128)); + z27 = _mm512_permutex2var_epi32(z25, z27, z26); + z16 = _mm512_cvtepu32_epi64(_mm512_castsi512_si256(z27)); + z27 = _mm512_loadu_si512((const void*)WC_PR(r14, 192)); + z27 = _mm512_permutex2var_epi32(z25, z27, z26); + z17 = _mm512_cvtepu32_epi64(_mm512_castsi512_si256(z27)); + z18 = _mm512_loadu_si512((const void*)WC_PR(r13, 0)); + z15 = _mm512_slli_epi64(z15, 6); + z16 = _mm512_slli_epi64(z16, 12); + z17 = _mm512_slli_epi64(z17, 18); + z19 = _mm512_mul_epu32(z4, z10); + z14 = _mm512_add_epi64(z19, z14); + z19 = _mm512_mul_epu32(z3, z11); + z20 = _mm512_mul_epu32(z4, z11); + z15 = _mm512_add_epi64(z20, z15); + z20 = _mm512_mul_epu32(z3, z12); + z21 = _mm512_mul_epu32(z4, z12); + z16 = _mm512_add_epi64(z21, z16); + z14 = _mm512_add_epi64(z19, z14); + z21 = _mm512_mul_epu32(z2, z12); + z22 = _mm512_mul_epu32(z4, z13); + z17 = _mm512_add_epi64(z22, z17); + z15 = _mm512_add_epi64(z20, z15); + z22 = _mm512_mul_epu32(z1, z13); + z19 = _mm512_mul_epu32(z2, z13); + z14 = _mm512_add_epi64(z21, z14); + z20 = _mm512_mul_epu32(z3, z13); + z21 = _mm512_mul_epu32(z3, z5); + z14 = _mm512_add_epi64(z22, z14); + z22 = _mm512_mul_epu32(z4, z5); + z18 = _mm512_add_epi64(z22, z18); + z15 = _mm512_add_epi64(z19, z15); + z22 = _mm512_mul_epu32(z0, z5); + z16 = _mm512_add_epi64(z20, z16); + z19 = _mm512_mul_epu32(z1, z5); + z17 = _mm512_add_epi64(z21, z17); + z20 = _mm512_mul_epu32(z2, z5); + z21 = _mm512_mul_epu32(z2, z6); + z14 = _mm512_add_epi64(z22, z14); + z22 = _mm512_mul_epu32(z3, z6); + z15 = _mm512_add_epi64(z19, z15); + z19 = _mm512_mul_epu32(z0, z6); + z16 = _mm512_add_epi64(z20, z16); + z20 = _mm512_mul_epu32(z1, z6); + z17 = _mm512_add_epi64(z21, z17); + z21 = _mm512_mul_epu32(z1, z7); + z18 = _mm512_add_epi64(z22, z18); + z22 = _mm512_mul_epu32(z2, z7); + z15 = _mm512_add_epi64(z19, z15); + z19 = _mm512_mul_epu32(z0, z7); + z16 = _mm512_add_epi64(z20, z16); + z20 = _mm512_mul_epu32(z0, z8); + z17 = _mm512_add_epi64(z21, z17); + z21 = _mm512_mul_epu32(z1, z8); + z18 = _mm512_add_epi64(z22, z18); + z16 = _mm512_add_epi64(z19, z16); + z22 = _mm512_mul_epu32(z0, z9); + z17 = _mm512_add_epi64(z20, z17); + z18 = _mm512_add_epi64(z21, z18); + z18 = _mm512_add_epi64(z22, z18); + z19 = _mm512_srli_epi64(z14, 26); + z20 = _mm512_srli_epi64(z17, 26); + z14 = _mm512_and_si512(z14, z23); + z17 = _mm512_and_si512(z17, z23); + z15 = _mm512_add_epi64(z19, z15); + z18 = _mm512_add_epi64(z20, z18); + z19 = _mm512_srli_epi64(z15, 26); + z20 = _mm512_srli_epi64(z18, 26); + z1 = _mm512_and_si512(z15, z23); + z4 = _mm512_and_si512(z18, z23); + z16 = _mm512_add_epi64(z19, z16); + z21 = _mm512_slli_epi32(z20, 2); + z21 = _mm512_add_epi32(z20, z21); + z19 = _mm512_srli_epi64(z16, 26); + z14 = _mm512_add_epi64(z21, z14); + z20 = _mm512_srli_epi64(z14, 26); + z2 = _mm512_and_si512(z16, z23); + z0 = _mm512_and_si512(z14, z23); + z17 = _mm512_add_epi64(z19, z17); + z1 = _mm512_add_epi64(z20, z1); + z19 = _mm512_srli_epi64(z17, 26); + z3 = _mm512_and_si512(z17, z23); + z4 = _mm512_add_epi64(z19, z4); + rsi = (word64)(rsi + 0x80); + rdx = (word64)(rdx - 0x80); + if ((rdx) != (0)) { + goto L_poly1305_avx512_blocks_start; + } +L_poly1305_avx512_blocks_store: + /* Store eight H values - state */ + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), _mm512_cvtepi64_epi32(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), _mm512_cvtepi64_epi32(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), _mm512_cvtepi64_epi32(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), _mm512_cvtepi64_epi32(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), _mm512_cvtepi64_epi32(z4)); +L_poly1305_avx512_blocks_end_calc: + if ((WC_L8(rdi, 616)) == (0)) { + goto L_poly1305_avx512_blocks_complete; + } + rax = (word64)(r8); + rdx = (word64)(r10); + rcx = (word64)(r12); + rdx = (word64)(rdx >> 12); + rcx = (word64)(rcx >> 24); + r9 = (word64)(r9 << 26); + r10 = (word64)(r10 << 52); + r11 = (word64)(r11 << 14); + r12 = (word64)(r12 << 40); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rax, r10, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rdx, r12, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(rcx); + rcx = (word64)(rcx & 3); + r8 = (word64)(r8 >> 2); + r8 = (word64)(r8 + r8 * 4); + cf = _addcarry_u64(0, rax, r8, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 24) = (word64)(rax); + WC_S64(rdi, 32) = (word64)(rdx); + WC_S64(rdi, 40) = (word64)(rcx); +L_poly1305_avx512_blocks_complete: + WC_S8(rdi, 617) = (byte)(1); + (void)z24; +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void poly1305_final_avx512(Poly1305* ctx, byte* mac) +{ + word64 rdi, rsi, rcx = 0, rsp, rdx = 0, rax = 0, r8 = 0; + __m256i y0 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)mac; + + rsp = (word64)(size_t)stk + 32; + WC_S8(rdi, 616) = (byte)(1); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, 617)) & 0xff); + if (((byte)rcx) == (0)) { + goto L_poly1305_avx512_final_done_blocks_X8; + } + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rsi; + rdx = (word64)(0x80); + rsi = (word64)(0); + (void)poly1305_blocks_avx512((Poly1305*)(size_t)rdi, (const unsigned char*)( + size_t)rsi, (size_t)rdx); + rsi = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); +L_poly1305_avx512_final_done_blocks_X8: + rax = (word64)(WC_L64(rdi, 608)); + rcx = (word64)(rax); + rcx = (word64)(rcx & -16); + if (((byte)rcx) == (0)) { + goto L_poly1305_avx512_final_done_blocks; + } + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rcx; + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rax; + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rsi; + rdx = (word64)(rcx); + rsi = (word64)(rdi + 480); + (void)poly1305_blocks_avx((Poly1305*)(size_t)rdi, (const unsigned char*)( + size_t)rsi, (size_t)rdx); + rsi = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); + rax = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); + rcx = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); +L_poly1305_avx512_final_done_blocks: + WC_S64(rdi, 608) = (word64)(WC_L64(rdi, 608) - rcx); + rdx = (word64)(0); + goto L_poly1305_avx512_final_cmp_copy; +L_poly1305_avx512_final_start_copy: + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, rcx + 480)) & 0xff); + WC_S8(rdi, rdx + 480) = (byte)((byte)r8); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx + 1) & 0xff); + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)((byte)rdx + 1) & 0xff); +L_poly1305_avx512_final_cmp_copy: + if (((byte)rax) != ((byte)rcx)) { + goto L_poly1305_avx512_final_start_copy; + } + (void)poly1305_final_avx((Poly1305*)(size_t)rdi, (byte*)(size_t)rsi); + y0 = _mm256_zextsi128_si256(_mm_setzero_si128()); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 624), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 656), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 688), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 720), y0); + WC_S64(rdi, 608) = (word64)(0); + WC_S16(rdi, 616) = (word16)(0); +} + +#endif /* HAVE_INTEL_AVX512 */ +#ifdef HAVE_INTEL_AVX512 +WOLFSSL_LOCAL void poly1305_calc_powers_avx512ifma(Poly1305* ctx) +{ + word64 rdi, rsp, rcx, r8, r9, rax, rdx, rsi, r13, r11, r12, r10, r15, r14, + rbx; + XALIGNED(32) WC_X64I_SLOT stk[12]; + unsigned char cf; + + rdi = (word64)(size_t)ctx; + + rsp = (word64)(size_t)stk + 96; + rsp = (word64)(rsp - 96); + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(0); + /* Convert to 44 bits in 64 */ + rax = (word64)(rcx); + rax = (word64)(rax << 20); + rax = (word64)(rax >> 20); + rdx = (word64)(rcx); + rdx = (word64)((rdx >> 44) | (r8 << 20)); + rdx = (word64)(rdx << 20); + rdx = (word64)(rdx >> 20); + rsi = (word64)(r8); + rsi = (word64)((rsi >> 24) | (r9 << 40)); + WC_S64(rdi, 224) = (word64)(rax); + WC_S64(rdi, 232) = (word64)(rdx); + WC_S64(rdi, 240) = (word64)(rsi); + WC_S64(rsp, 0) = (word64)(rcx); + WC_S64(rsp, 8) = (word64)(r8); + WC_S64(rsp, 16) = (word64)(r9); + /* Square 128-bit */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, rax); + r10 = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r15, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* Reduce 256-bit to 130-bit */ + rax = (word64)(r12); + rdx = (word64)(r13); + rax = (word64)(rax & -4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)(rdx >> 2); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + rax = (word64)(r12); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* Convert to 44 bits in 64 */ + rax = (word64)(r10); + rax = (word64)(rax << 20); + rax = (word64)(rax >> 20); + rdx = (word64)(r10); + rdx = (word64)((rdx >> 44) | (r11 << 20)); + rdx = (word64)(rdx << 20); + rdx = (word64)(rdx >> 20); + rsi = (word64)(r11); + rsi = (word64)((rsi >> 24) | (r12 << 40)); + WC_S64(rdi, 256) = (word64)(rax); + WC_S64(rdi, 264) = (word64)(rdx); + WC_S64(rdi, 272) = (word64)(rsi); + WC_S64(rsp, 24) = (word64)(r10); + WC_S64(rsp, 32) = (word64)(r11); + WC_S64(rsp, 40) = (word64)(r12); + /* Multiply 128-bit by 130-bit */ + /* r1[0] * r2[0] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r10); + r13 = (word64)(rax); + r14 = (word64)(rdx); + /* r1[0] * r2[1] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r11); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* r1[1] * r2[0] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r10); + rsi = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + /* r1[0] * r2[2] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + /* r1[1] * r2[1] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r11); + rbx = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[2] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r15); + rdx = (word64)(rsi); + rbx = (word64)(rbx); + rax = (word64)(rax & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (rbx << 62)); + rbx = (word64)(rbx >> 2); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Convert to 44 bits in 64 */ + rax = (word64)(r13); + rax = (word64)(rax << 20); + rax = (word64)(rax >> 20); + rdx = (word64)(r13); + rdx = (word64)((rdx >> 44) | (r14 << 20)); + rdx = (word64)(rdx << 20); + rdx = (word64)(rdx >> 20); + rsi = (word64)(r14); + rsi = (word64)((rsi >> 24) | (r15 << 40)); + WC_S64(rdi, 288) = (word64)(rax); + WC_S64(rdi, 296) = (word64)(rdx); + WC_S64(rdi, 304) = (word64)(rsi); + WC_S64(rsp, 48) = (word64)(r13); + WC_S64(rsp, 56) = (word64)(r14); + WC_S64(rsp, 64) = (word64)(r15); + /* Square 130-bit */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r10); + r13 = (word64)(0); + r8 = (word64)(rax); + r9 = (word64)(rdx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, rax); + rcx = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, r15, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, rax); + r14 = (word64)(rax); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r10); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(r12); + WC_X64I_MUL128(rax, rdx, rax, r11); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r9); + rdx = (word64)(r13); + r15 = (word64)(r14); + rax = (word64)(rax & -4); + r9 = (word64)(r9 & 3); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r15, (unsigned long long*)&r9); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (r15 << 62)); + r15 = (word64)(r15 >> 2); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r15, (unsigned long long*)&r9); + rax = (word64)(r9); + r9 = (word64)(r9 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* Convert to 44 bits in 64 */ + rax = (word64)(rcx); + rax = (word64)(rax << 20); + rax = (word64)(rax >> 20); + rdx = (word64)(rcx); + rdx = (word64)((rdx >> 44) | (r8 << 20)); + rdx = (word64)(rdx << 20); + rdx = (word64)(rdx >> 20); + rsi = (word64)(r8); + rsi = (word64)((rsi >> 24) | (r9 << 40)); + WC_S64(rdi, 320) = (word64)(rax); + WC_S64(rdi, 328) = (word64)(rdx); + WC_S64(rdi, 336) = (word64)(rsi); + WC_S64(rsp, 72) = (word64)(rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r10 = (word64)(WC_L64(rsp, 0)); + r11 = (word64)(WC_L64(rsp, 8)); + r12 = (word64)(WC_L64(rsp, 16)); + /* Multiply 128-bit by 130-bit */ + /* r1[0] * r2[0] */ + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(rax); + r14 = (word64)(rdx); + /* r1[0] * r2[1] */ + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, r8); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* r1[1] * r2[0] */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, rcx); + rsi = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + /* r1[0] * r2[2] */ + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, r9); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + /* r1[1] * r2[1] */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r8); + rbx = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[2] */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, r9); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r15); + rdx = (word64)(rsi); + rbx = (word64)(rbx); + rax = (word64)(rax & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (rbx << 62)); + rbx = (word64)(rbx >> 2); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Convert to 44 bits in 64 */ + rax = (word64)(r13); + rax = (word64)(rax << 20); + rax = (word64)(rax >> 20); + rdx = (word64)(r13); + rdx = (word64)((rdx >> 44) | (r14 << 20)); + rdx = (word64)(rdx << 20); + rdx = (word64)(rdx >> 20); + rsi = (word64)(r14); + rsi = (word64)((rsi >> 24) | (r15 << 40)); + WC_S64(rdi, 624) = (word64)(rax); + WC_S64(rdi, 632) = (word64)(rdx); + WC_S64(rdi, 640) = (word64)(rsi); + /* Square 130-bit */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, rax); + r10 = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r15, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rax); + r14 = (word64)(rax); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, r8); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r12); + rdx = (word64)(r13); + r15 = (word64)(r14); + rax = (word64)(rax & -4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (r15 << 62)); + r15 = (word64)(r15 >> 2); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)(r12); + r12 = (word64)(r12 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* Convert to 44 bits in 64 */ + rax = (word64)(r10); + rax = (word64)(rax << 20); + rax = (word64)(rax >> 20); + rdx = (word64)(r10); + rdx = (word64)((rdx >> 44) | (r11 << 20)); + rdx = (word64)(rdx << 20); + rdx = (word64)(rdx >> 20); + rsi = (word64)(r11); + rsi = (word64)((rsi >> 24) | (r12 << 40)); + WC_S64(rdi, 720) = (word64)(rax); + WC_S64(rdi, 728) = (word64)(rdx); + WC_S64(rdi, 736) = (word64)(rsi); + rcx = (word64)(WC_L64(rsp, 48)); + r8 = (word64)(WC_L64(rsp, 56)); + r9 = (word64)(WC_L64(rsp, 64)); + /* Square 130-bit */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, rax); + r10 = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r15, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rax); + r14 = (word64)(rax); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, r8); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r12); + rdx = (word64)(r13); + r15 = (word64)(r14); + rax = (word64)(rax & -4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (r15 << 62)); + r15 = (word64)(r15 >> 2); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)(r12); + r12 = (word64)(r12 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* Convert to 44 bits in 64 */ + rax = (word64)(r10); + rax = (word64)(rax << 20); + rax = (word64)(rax >> 20); + rdx = (word64)(r10); + rdx = (word64)((rdx >> 44) | (r11 << 20)); + rdx = (word64)(rdx << 20); + rdx = (word64)(rdx >> 20); + rsi = (word64)(r11); + rsi = (word64)((rsi >> 24) | (r12 << 40)); + WC_S64(rdi, 656) = (word64)(rax); + WC_S64(rdi, 664) = (word64)(rdx); + WC_S64(rdi, 672) = (word64)(rsi); + rcx = (word64)(WC_L64(rsp, 0)); + r8 = (word64)(WC_L64(rsp, 8)); + r9 = (word64)(WC_L64(rsp, 16)); + /* Multiply 128-bit by 130-bit */ + /* r1[0] * r2[0] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r10); + r13 = (word64)(rax); + r14 = (word64)(rdx); + /* r1[0] * r2[1] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r11); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* r1[1] * r2[0] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r10); + rsi = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + /* r1[0] * r2[2] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + /* r1[1] * r2[1] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r11); + rbx = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[2] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r15); + rdx = (word64)(rsi); + rbx = (word64)(rbx); + rax = (word64)(rax & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (rbx << 62)); + rbx = (word64)(rbx >> 2); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* Convert to 44 bits in 64 */ + rax = (word64)(r13); + rax = (word64)(rax << 20); + rax = (word64)(rax >> 20); + rdx = (word64)(r13); + rdx = (word64)((rdx >> 44) | (r14 << 20)); + rdx = (word64)(rdx << 20); + rdx = (word64)(rdx >> 20); + rsi = (word64)(r14); + rsi = (word64)((rsi >> 24) | (r15 << 40)); + WC_S64(rdi, 688) = (word64)(rax); + WC_S64(rdi, 696) = (word64)(rdx); + WC_S64(rdi, 704) = (word64)(rsi); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void poly1305_setkey_avx512ifma(Poly1305* ctx, const byte* key) +{ + word64 rdi, rsi; + __m512i z0; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)key; + + (void)poly1305_setkey_avx((Poly1305*)(size_t)rdi, (const byte*)(size_t)rsi); + z0 = _mm512_setzero_si512(); + _mm512_storeu_si512((void*)WC_PW(rdi, 752), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 816), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 880), z0); + WC_S64(rdi, 608) = (word64)(0); + WC_S16(rdi, 616) = (word16)(0); +} + +XALIGNED(32) static const word64 L_poly1305_avx512ifma_blocks_mask44[] + WC_X64I_UNUSED = { + 0x00000fffffffffffULL, 0x00000fffffffffffULL, + 0x00000fffffffffffULL, 0x00000fffffffffffULL, + 0x00000fffffffffffULL, 0x00000fffffffffffULL, + 0x00000fffffffffffULL, 0x00000fffffffffffULL, +}; + +XALIGNED(32) static const word64 L_poly1305_avx512ifma_blocks_mask24[] + WC_X64I_UNUSED = { + 0x0000000000ffffffULL, 0x0000000000ffffffULL, + 0x0000000000ffffffULL, 0x0000000000ffffffULL, + 0x0000000000ffffffULL, 0x0000000000ffffffULL, + 0x0000000000ffffffULL, 0x0000000000ffffffULL, +}; + +XALIGNED(32) static const word64 L_poly1305_avx512ifma_blocks_pad[] + WC_X64I_UNUSED = { + 0x0000010000000000ULL, 0x0000010000000000ULL, + 0x0000010000000000ULL, 0x0000010000000000ULL, + 0x0000010000000000ULL, 0x0000010000000000ULL, + 0x0000010000000000ULL, 0x0000010000000000ULL, +}; + +XALIGNED(32) static const word64 L_poly1305_avx512ifma_blocks_idxeven[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000004ULL, 0x0000000000000006ULL, + 0x0000000000000008ULL, 0x000000000000000aULL, + 0x000000000000000cULL, 0x000000000000000eULL, +}; + +XALIGNED(32) static const word64 L_poly1305_avx512ifma_blocks_idxodd[] + WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000003ULL, + 0x0000000000000005ULL, 0x0000000000000007ULL, + 0x0000000000000009ULL, 0x000000000000000bULL, + 0x000000000000000dULL, 0x000000000000000fULL, +}; + +XALIGNED(32) static const word64 L_poly1305_avx512ifma_blocks_rxidx[] + WC_X64I_UNUSED = { + 0x000000000000000cULL, 0x0000000000000008ULL, + 0x0000000000000004ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x000000000000000dULL, 0x0000000000000009ULL, + 0x0000000000000005ULL, 0x0000000000000001ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x000000000000000eULL, 0x000000000000000aULL, + 0x0000000000000006ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512ifma") +WOLFSSL_LOCAL void poly1305_blocks_avx512ifma(Poly1305* ctx, + const unsigned char* m, size_t bytes) +{ + word64 rdi, rsi, rdx, r8, r9, r10, r11, r12, r13, rax = 0, rcx = 0, + r14 = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), + z8 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), + z12 = _mm512_setzero_si512(), z13 = _mm512_setzero_si512(), + z14 = _mm512_setzero_si512(), z15 = _mm512_setzero_si512(), + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), z18; + unsigned char cf; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)m; + rdx = (word64)(word64)bytes; + + r8 = (word64)((word64)(size_t)L_poly1305_avx512ifma_blocks_mask44); + r9 = (word64)((word64)(size_t)L_poly1305_avx512ifma_blocks_mask24); + r10 = (word64)((word64)(size_t)L_poly1305_avx512ifma_blocks_pad); + r11 = (word64)((word64)(size_t)L_poly1305_avx512ifma_blocks_idxeven); + r12 = (word64)((word64)(size_t)L_poly1305_avx512ifma_blocks_idxodd); + r13 = (word64)((word64)(size_t)L_poly1305_avx512ifma_blocks_rxidx); + z18 = _mm512_loadu_si512((const void*)WC_PR(r8, 0)); + if ((WC_L16(rdi, 616)) != (0)) { + goto L_poly1305_avx512ifma_blocks_begin_h; + } + /* First 8 blocks initialise the lanes directly. */ + z14 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z16 = _mm512_loadu_si512((const void*)WC_PR(r11, 0)); + z16 = _mm512_permutex2var_epi64(z14, z16, z15); + z17 = _mm512_loadu_si512((const void*)WC_PR(r12, 0)); + z17 = _mm512_permutex2var_epi64(z14, z17, z15); + z0 = _mm512_and_si512(z16, _mm512_loadu_si512((const void*)WC_PR(r8, 0))); + z14 = _mm512_srli_epi64(z16, 44); + z15 = _mm512_and_si512(z17, _mm512_loadu_si512((const void*)WC_PR(r9, 0))); + z15 = _mm512_slli_epi64(z15, 20); + z1 = _mm512_or_si512(z15, z14); + z2 = _mm512_srli_epi64(z17, 24); + z2 = _mm512_or_si512(z2, _mm512_loadu_si512((const void*)WC_PR(r10, 0))); + rsi = (word64)(rsi + 0x80); + rdx = (word64)(rdx - 0x80); + if ((rdx) == (0)) { + goto L_poly1305_avx512ifma_blocks_store; + } + goto L_poly1305_avx512ifma_blocks_load_r8; +L_poly1305_avx512ifma_blocks_begin_h: + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 752)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 816)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 880)); + if ((WC_L8(rdi, 616)) == (0)) { + goto L_poly1305_avx512ifma_blocks_load_r8; + } + /* Finished: load the 8 distinct powers r^8..r^1. */ + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 224)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rdi, 288)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdi, 624)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 688)); + z12 = _mm512_loadu_si512((const void*)WC_PR(r13, 0)); + z12 = _mm512_permutex2var_epi64(z10, z12, z11); + z13 = _mm512_loadu_si512((const void*)WC_PR(r13, 0)); + z13 = _mm512_permutex2var_epi64(z8, z13, z9); + z3 = _mm512_inserti64x4(z12, _mm512_castsi512_si256(z13), 1); + z12 = _mm512_loadu_si512((const void*)WC_PR(r13, 64)); + z12 = _mm512_permutex2var_epi64(z10, z12, z11); + z13 = _mm512_loadu_si512((const void*)WC_PR(r13, 64)); + z13 = _mm512_permutex2var_epi64(z8, z13, z9); + z4 = _mm512_inserti64x4(z12, _mm512_castsi512_si256(z13), 1); + z12 = _mm512_loadu_si512((const void*)WC_PR(r13, 128)); + z12 = _mm512_permutex2var_epi64(z10, z12, z11); + z13 = _mm512_loadu_si512((const void*)WC_PR(r13, 128)); + z13 = _mm512_permutex2var_epi64(z8, z13, z9); + z5 = _mm512_inserti64x4(z12, _mm512_castsi512_si256(z13), 1); + goto L_poly1305_avx512ifma_blocks_do_mul; +L_poly1305_avx512ifma_blocks_load_r8: + z3 = _mm512_set1_epi64((long long)WC_L64(rdi, 720)); + z4 = _mm512_set1_epi64((long long)WC_L64(rdi, 728)); + z5 = _mm512_set1_epi64((long long)WC_L64(rdi, 736)); +L_poly1305_avx512ifma_blocks_do_mul: + z14 = _mm512_slli_epi64(z4, 2); + z6 = _mm512_slli_epi64(z4, 4); + z6 = _mm512_add_epi64(z6, z14); + z14 = _mm512_slli_epi64(z5, 2); + z7 = _mm512_slli_epi64(z5, 4); + z7 = _mm512_add_epi64(z7, z14); + if ((WC_L8(rdi, 616)) != (1)) { + goto L_poly1305_avx512ifma_blocks_start; + } + /* Finished: final multiply, collapse lanes, reduce to ctx->h. */ + z8 = _mm512_setzero_si512(); + z9 = _mm512_setzero_si512(); + z10 = _mm512_setzero_si512(); + z11 = _mm512_setzero_si512(); + z12 = _mm512_setzero_si512(); + z13 = _mm512_setzero_si512(); + z8 = _mm512_madd52lo_epu64(z8, z0, z3); + z8 = _mm512_madd52lo_epu64(z8, z1, z7); + z8 = _mm512_madd52lo_epu64(z8, z2, z6); + z11 = _mm512_madd52hi_epu64(z11, z0, z3); + z11 = _mm512_madd52hi_epu64(z11, z1, z7); + z11 = _mm512_madd52hi_epu64(z11, z2, z6); + z9 = _mm512_madd52lo_epu64(z9, z0, z4); + z9 = _mm512_madd52lo_epu64(z9, z1, z3); + z9 = _mm512_madd52lo_epu64(z9, z2, z7); + z12 = _mm512_madd52hi_epu64(z12, z0, z4); + z12 = _mm512_madd52hi_epu64(z12, z1, z3); + z12 = _mm512_madd52hi_epu64(z12, z2, z7); + z10 = _mm512_madd52lo_epu64(z10, z0, z5); + z10 = _mm512_madd52lo_epu64(z10, z1, z4); + z10 = _mm512_madd52lo_epu64(z10, z2, z3); + z13 = _mm512_madd52hi_epu64(z13, z0, z5); + z13 = _mm512_madd52hi_epu64(z13, z1, z4); + z13 = _mm512_madd52hi_epu64(z13, z2, z3); + z14 = _mm512_slli_epi64(z11, 8); + z9 = _mm512_add_epi64(z14, z9); + z14 = _mm512_slli_epi64(z12, 8); + z10 = _mm512_add_epi64(z14, z10); + z15 = _mm512_slli_epi64(z13, 8); + z14 = _mm512_slli_epi64(z15, 2); + z15 = _mm512_slli_epi64(z15, 4); + z15 = _mm512_add_epi64(z15, z14); + z8 = _mm512_add_epi64(z15, z8); + z14 = _mm512_srli_epi64(z8, 44); + z0 = _mm512_and_si512(z8, z18); + z9 = _mm512_add_epi64(z14, z9); + z14 = _mm512_srli_epi64(z9, 44); + z1 = _mm512_and_si512(z9, z18); + z10 = _mm512_add_epi64(z14, z10); + z14 = _mm512_srli_epi64(z10, 44); + z2 = _mm512_and_si512(z10, z18); + z15 = _mm512_slli_epi64(z14, 2); + z14 = _mm512_slli_epi64(z14, 4); + z14 = _mm512_add_epi64(z14, z15); + z0 = _mm512_add_epi64(z14, z0); + z14 = _mm512_shuffle_i64x2(z0, z0, 0x4e); + z15 = _mm512_shuffle_i64x2(z1, z1, 0x4e); + z16 = _mm512_shuffle_i64x2(z2, z2, 0x4e); + z0 = _mm512_add_epi64(z14, z0); + z1 = _mm512_add_epi64(z15, z1); + z2 = _mm512_add_epi64(z16, z2); + z14 = _mm512_bsrli_epi128(z0, 8); + z15 = _mm512_bsrli_epi128(z1, 8); + z16 = _mm512_bsrli_epi128(z2, 8); + z0 = _mm512_add_epi64(z14, z0); + z1 = _mm512_add_epi64(z15, z1); + z2 = _mm512_add_epi64(z16, z2); + z14 = _mm512_permutex_epi64(z0, 2); + z15 = _mm512_permutex_epi64(z1, 2); + z16 = _mm512_permutex_epi64(z2, 2); + z0 = _mm512_add_epi64(z14, z0); + z1 = _mm512_add_epi64(z15, z1); + z2 = _mm512_add_epi64(z16, z2); + rax = (word64)((word64)_mm_cvtsi128_si64(_mm512_castsi512_si128(z0))); + rcx = (word64)((word64)_mm_cvtsi128_si64(_mm512_castsi512_si128(z1))); + rdx = (word64)((word64)_mm_cvtsi128_si64(_mm512_castsi512_si128(z2))); + r9 = (word64)(rcx); + r9 = (word64)(r9 >> 20); + r10 = (word64)(rdx); + r10 = (word64)(r10 >> 40); + r8 = (word64)(rax); + rax = (word64)(rcx); + rax = (word64)(rax << 44); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + rax = (word64)(rdx); + rax = (word64)(rax << 24); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r14 = (word64)(r10); + r10 = (word64)(r10 & 3); + r14 = (word64)(r14 >> 2); + r14 = (word64)(r14 + r14 * 4); + cf = _addcarry_u64(0, r8, r14, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); + goto L_poly1305_avx512ifma_blocks_end_calc; +L_poly1305_avx512ifma_blocks_start: + z14 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z16 = _mm512_loadu_si512((const void*)WC_PR(r11, 0)); + z16 = _mm512_permutex2var_epi64(z14, z16, z15); + z17 = _mm512_loadu_si512((const void*)WC_PR(r12, 0)); + z17 = _mm512_permutex2var_epi64(z14, z17, z15); + z8 = _mm512_and_si512(z16, _mm512_loadu_si512((const void*)WC_PR(r8, 0))); + z14 = _mm512_srli_epi64(z16, 44); + z15 = _mm512_and_si512(z17, _mm512_loadu_si512((const void*)WC_PR(r9, 0))); + z15 = _mm512_slli_epi64(z15, 20); + z9 = _mm512_or_si512(z15, z14); + z10 = _mm512_srli_epi64(z17, 24); + z10 = _mm512_or_si512(z10, _mm512_loadu_si512((const void*)WC_PR(r10, 0))); + z11 = _mm512_setzero_si512(); + z12 = _mm512_setzero_si512(); + z13 = _mm512_setzero_si512(); + z8 = _mm512_madd52lo_epu64(z8, z0, z3); + z8 = _mm512_madd52lo_epu64(z8, z1, z7); + z8 = _mm512_madd52lo_epu64(z8, z2, z6); + z11 = _mm512_madd52hi_epu64(z11, z0, z3); + z11 = _mm512_madd52hi_epu64(z11, z1, z7); + z11 = _mm512_madd52hi_epu64(z11, z2, z6); + z9 = _mm512_madd52lo_epu64(z9, z0, z4); + z9 = _mm512_madd52lo_epu64(z9, z1, z3); + z9 = _mm512_madd52lo_epu64(z9, z2, z7); + z12 = _mm512_madd52hi_epu64(z12, z0, z4); + z12 = _mm512_madd52hi_epu64(z12, z1, z3); + z12 = _mm512_madd52hi_epu64(z12, z2, z7); + z10 = _mm512_madd52lo_epu64(z10, z0, z5); + z10 = _mm512_madd52lo_epu64(z10, z1, z4); + z10 = _mm512_madd52lo_epu64(z10, z2, z3); + z13 = _mm512_madd52hi_epu64(z13, z0, z5); + z13 = _mm512_madd52hi_epu64(z13, z1, z4); + z13 = _mm512_madd52hi_epu64(z13, z2, z3); + z14 = _mm512_slli_epi64(z11, 8); + z9 = _mm512_add_epi64(z14, z9); + z14 = _mm512_slli_epi64(z12, 8); + z10 = _mm512_add_epi64(z14, z10); + z15 = _mm512_slli_epi64(z13, 8); + z14 = _mm512_slli_epi64(z15, 2); + z15 = _mm512_slli_epi64(z15, 4); + z15 = _mm512_add_epi64(z15, z14); + z8 = _mm512_add_epi64(z15, z8); + z14 = _mm512_srli_epi64(z8, 44); + z0 = _mm512_and_si512(z8, z18); + z9 = _mm512_add_epi64(z14, z9); + z14 = _mm512_srli_epi64(z9, 44); + z1 = _mm512_and_si512(z9, z18); + z10 = _mm512_add_epi64(z14, z10); + z14 = _mm512_srli_epi64(z10, 44); + z2 = _mm512_and_si512(z10, z18); + z15 = _mm512_slli_epi64(z14, 2); + z14 = _mm512_slli_epi64(z14, 4); + z14 = _mm512_add_epi64(z14, z15); + z0 = _mm512_add_epi64(z14, z0); + rsi = (word64)(rsi + 0x80); + rdx = (word64)(rdx - 0x80); + if ((rdx) != (0)) { + goto L_poly1305_avx512ifma_blocks_start; + } +L_poly1305_avx512ifma_blocks_store: + _mm512_storeu_si512((void*)WC_PW(rdi, 752), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 816), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 880), z2); +L_poly1305_avx512ifma_blocks_end_calc: + WC_S8(rdi, 617) = (byte)(1); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void poly1305_final_avx512ifma(Poly1305* ctx, byte* mac) +{ + word64 rdi, rsi, rcx = 0, rsp, rdx = 0, rax = 0, r8 = 0; + __m512i z0 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[4]; + + rdi = (word64)(size_t)ctx; + rsi = (word64)(size_t)mac; + + rsp = (word64)(size_t)stk + 32; + WC_S8(rdi, 616) = (byte)(1); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, 617)) & 0xff); + if (((byte)rcx) == (0)) { + goto L_poly1305_avx512ifma_final_done_blocks_8; + } + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rsi; + rdx = (word64)(0x80); + rsi = (word64)(0); + (void)poly1305_blocks_avx512ifma((Poly1305*)(size_t)rdi, ( + const unsigned char*)(size_t)rsi, (size_t)rdx); + rsi = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); +L_poly1305_avx512ifma_final_done_blocks_8: + rax = (word64)(WC_L64(rdi, 608)); + rcx = (word64)(rax); + rcx = (word64)(rcx & -16); + if (((byte)rcx) == (0)) { + goto L_poly1305_avx512ifma_final_done_blocks; + } + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rcx; + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rax; + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rsi; + rdx = (word64)(rcx); + rsi = (word64)(rdi + 480); + (void)poly1305_blocks_avx((Poly1305*)(size_t)rdi, (const unsigned char*)( + size_t)rsi, (size_t)rdx); + rsi = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); + rax = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); + rcx = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); +L_poly1305_avx512ifma_final_done_blocks: + WC_S64(rdi, 608) = (word64)(WC_L64(rdi, 608) - rcx); + rdx = (word64)(0); + goto L_poly1305_avx512ifma_final_cmp_copy; +L_poly1305_avx512ifma_final_start_copy: + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, rcx + 480)) & 0xff); + WC_S8(rdi, rdx + 480) = (byte)((byte)r8); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx + 1) & 0xff); + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)((byte)rdx + 1) & 0xff); +L_poly1305_avx512ifma_final_cmp_copy: + if (((byte)rax) != ((byte)rcx)) { + goto L_poly1305_avx512ifma_final_start_copy; + } + (void)poly1305_final_avx((Poly1305*)(size_t)rdi, (byte*)(size_t)rsi); + z0 = _mm512_setzero_si512(); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 624), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 656), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 688), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 720), _mm512_castsi512_si256(z0)); + _mm512_storeu_si512((void*)WC_PW(rdi, 752), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 816), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 880), z0); + WC_S64(rdi, 608) = (word64)(0); + WC_S16(rdi, 616) = (word16)(0); +} + +WOLFSSL_LOCAL void poly1305_fold_avx512ifma(Poly1305* ctx, word32 nBlocks) +{ + word64 rdi, rsp, rsi, rcx, r8, r9, rax, rdx = 0, r13 = 0, r11 = 0, + r12 = 0, r10 = 0, r15 = 0, r14 = 0, rbx = 0, rbp = 0; + XALIGNED(32) WC_X64I_SLOT stk[8]; + unsigned char cf; + + rdi = (word64)(size_t)ctx; + + rsp = (word64)(size_t)stk + 64; + rsi = (word64)(word32)nBlocks; + rsp = (word64)(rsp - 64); + WC_S32(rsp, 24) = (word32)((word32)rsi); + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rcx); + WC_S64(rsp, 8) = (word64)(r8); + WC_S64(rsp, 16) = (word64)(r9); + rax = (word64)(WC_X64I_BSR64(rsi)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + WC_S64(rsp, 32) = (word64)(rax); +L_poly1305_fold_loop: + rax = (word64)(WC_L64(rsp, 32)); + if (((rax & rax)) == (0)) { + goto L_poly1305_fold_done; + } + rax = (word64)(rax - 1); + WC_S64(rsp, 32) = (word64)(rax); + rcx = (word64)(WC_L64(rsp, 0)); + r8 = (word64)(WC_L64(rsp, 8)); + r9 = (word64)(WC_L64(rsp, 16)); + /* Square 130-bit */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rcx); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, rax); + r10 = (word64)(rax); + r15 = (word64)(rdx); + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r15, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rax); + r14 = (word64)(rax); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, rcx); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, r8); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r12); + rdx = (word64)(r13); + r15 = (word64)(r14); + rax = (word64)(rax & -4); + r12 = (word64)(r12 & 3); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (r15 << 62)); + r15 = (word64)(r15 >> 2); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r15, (unsigned long long*)&r12); + rax = (word64)(r12); + r12 = (word64)(r12 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 0) = (word64)(r10); + WC_S64(rsp, 8) = (word64)(r11); + WC_S64(rsp, 16) = (word64)(r12); + rax = (word32)(WC_L32(rsp, 24)); + rdx = (word64)(WC_L64(rsp, 32)); + if (((unsigned char)((rax >> (rdx & 63)) & 1)) == 0) { + goto L_poly1305_fold_skip; + } + rcx = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)(0); + r10 = (word64)(WC_L64(rsp, 0)); + r11 = (word64)(WC_L64(rsp, 8)); + r12 = (word64)(WC_L64(rsp, 16)); + /* Multiply 128-bit by 130-bit */ + /* r1[0] * r2[0] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r10); + r13 = (word64)(rax); + r14 = (word64)(rdx); + /* r1[0] * r2[1] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r11); + r15 = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* r1[1] * r2[0] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r10); + rsi = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + /* r1[0] * r2[2] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + /* r1[1] * r2[1] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r11); + rbx = (word64)(0); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[2] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r15); + rdx = (word64)(rsi); + rbx = (word64)(rbx); + rax = (word64)(rax & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (rbx << 62)); + rbx = (word64)(rbx >> 2); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbx, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + WC_S64(rsp, 0) = (word64)(r13); + WC_S64(rsp, 8) = (word64)(r14); + WC_S64(rsp, 16) = (word64)(r15); +L_poly1305_fold_skip: + goto L_poly1305_fold_loop; +L_poly1305_fold_done: + rcx = (word64)(WC_L64(rdi, 64)); + r8 = (word64)(WC_L64(rdi, 72)); + r9 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(WC_L64(rsp, 0)); + r11 = (word64)(WC_L64(rsp, 8)); + r12 = (word64)(WC_L64(rsp, 16)); + /* Multiply 130-bit by 130-bit */ + r13 = (word64)(0); + r14 = (word64)(0); + r15 = (word64)(0); + rsi = (word64)(0); + rbx = (word64)(0); + /* r1[0] * r2[0] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r10); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[0] * r2[1] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r11); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[0] * r2[2] */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[0] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r10); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[1] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r11); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[1] * r2[2] */ + rax = (word64)(r8); + WC_X64I_MUL128(rax, rdx, rax, r12); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* r1[2] * r2[0] */ + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, r10); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, rdx, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* r1[2] * r2[1] */ + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, r11); + cf = _addcarry_u64(0, rsi, rax, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* r1[2] * r2[2] */ + rax = (word64)(r9); + WC_X64I_MUL128(rax, rdx, rax, r12); + rbx = (word64)(rbx + rax); + /* Reduce 260-bit to 130-bit */ + rax = (word64)(r15); + rdx = (word64)(rsi); + rbp = (word64)(rbx); + rax = (word64)(rax & -4); + r15 = (word64)(r15 & 3); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbp, (unsigned long long*)&r15); + rax = (word64)((rax >> 2) | (rdx << 62)); + rdx = (word64)((rdx >> 2) | (rbp << 62)); + rbp = (word64)(rbp >> 2); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rbp, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + rax = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + rax = (word64)(WC_L64(rdi, 32)); + cf = _addcarry_u64(cf, r14, rax, (unsigned long long*)&r14); + rax = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(cf, r15, rax, (unsigned long long*)&r15); + rax = (word64)(r15); + r15 = (word64)(r15 & 3); + rax = (word64)(rax >> 2); + rax = (word64)(rax + rax * 4); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rdi, 32) = (word64)(r14); + WC_S64(rdi, 40) = (word64)(r15); +} + +#endif /* HAVE_INTEL_AVX512 */ +#endif /* WOLFSSL_X86_64_BUILD */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/sha256_asm.asm b/wolfcrypt/src/sha256_asm.asm index 4fa488d79e8..be9750ab763 100644 --- a/wolfcrypt/src/sha256_asm.asm +++ b/wolfcrypt/src/sha256_asm.asm @@ -12952,7 +12952,9 @@ Transform_Sha256_AVX2_Len PROC vmovdqu ymm1, YMMWORD PTR [rsi+32] vmovups YMMWORD PTR [rdi+32], ymm0 vmovups YMMWORD PTR [rdi+64], ymm1 + sub rsp, 32 call Transform_Sha256_AVX2 + add rsp, 32 add rsi, 64 sub DWORD PTR [rsp+512], 64 jz L_sha256_len_avx2_done @@ -19521,7 +19523,9 @@ Transform_Sha256_AVX2_RORX_Len PROC vmovdqu ymm1, YMMWORD PTR [rsi+32] vmovups YMMWORD PTR [rdi+32], ymm0 vmovups YMMWORD PTR [rdi+64], ymm1 + sub rsp, 32 call Transform_Sha256_AVX2_RORX + add rsp, 32 add rsi, 64 sub DWORD PTR [rsp+512], 64 jz L_sha256_len_avx2_rorx_done diff --git a/wolfcrypt/src/sha256_x86_64_intrin.c b/wolfcrypt/src/sha256_x86_64_intrin.c new file mode 100644 index 00000000000..db164ce8c7b --- /dev/null +++ b/wolfcrypt/src/sha256_x86_64_intrin.c @@ -0,0 +1,23636 @@ +/* sha256_x86_64_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_SHA256_X86_64_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_X86_64_BUILD +extern WOLFSSL_LOCAL int Transform_Sha256_SSE2_Sha(wc_Sha256* sha256, + const byte* data); +extern WOLFSSL_LOCAL int Transform_Sha256_SSE2_Sha_Len(wc_Sha256* sha256, + const byte* data, word32 len); +#ifdef HAVE_INTEL_AVX1 +extern WOLFSSL_LOCAL int Transform_Sha256_AVX1(wc_Sha256* sha256, + const byte* data); +extern WOLFSSL_LOCAL int Transform_Sha256_AVX1_Len(wc_Sha256* sha256, + const byte* data, word32 len); +extern WOLFSSL_LOCAL int Transform_Sha256_AVX1_RORX(wc_Sha256* sha256, + const byte* data); +extern WOLFSSL_LOCAL int Transform_Sha256_AVX1_RORX_Len(wc_Sha256* sha256, + const byte* data, word32 len); +extern WOLFSSL_LOCAL int Transform_Sha256_AVX1_Sha(wc_Sha256* sha256, + const byte* data); +extern WOLFSSL_LOCAL int Transform_Sha256_AVX1_Sha_Len(wc_Sha256* sha256, + const byte* data, word32 len); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL int Transform_Sha256_AVX2(wc_Sha256* sha256, + const byte* data); +extern WOLFSSL_LOCAL int Transform_Sha256_AVX2_Len(wc_Sha256* sha256, + const byte* data, word32 len); +extern WOLFSSL_LOCAL int Transform_Sha256_AVX2_RORX(wc_Sha256* sha256, + const byte* data); +extern WOLFSSL_LOCAL int Transform_Sha256_AVX2_RORX_Len(wc_Sha256* sha256, + const byte* data, word32 len); + +#endif +#endif +#ifdef WOLFSSL_X86_64_BUILD +XALIGNED(16) static const word32 L_sse2_sha256_sha_k[] WC_X64I_UNUSED = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +XALIGNED(16) static const word64 L_sse2_sha256_shuf_mask[] WC_X64I_UNUSED = { + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, +}; + +WC_X64I_TARGET("sse2,ssse3,sha") +WOLFSSL_LOCAL int Transform_Sha256_SSE2_Sha(wc_Sha256* sha256, const byte* data) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + + rdx = (word64)((word64)(size_t)L_sse2_sha256_sha_k); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(L_sse2_sha256_shuf_mask, 0)); + x1 = _mm_loadl_epi64((const __m128i*)WC_PR(rdi, 0)); + x2 = _mm_loadl_epi64((const __m128i*)WC_PR(rdi, 8)); + x1 = _mm_unpacklo_epi64(x1, _mm_loadl_epi64((const __m128i*)WC_PR(rdi, + 16))); + x2 = _mm_unpacklo_epi64(x2, _mm_loadl_epi64((const __m128i*)WC_PR(rdi, + 24))); + x1 = _mm_shuffle_epi32(x1, 0x1b); + x2 = _mm_shuffle_epi32(x2, 0x1b); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x3 = _mm_shuffle_epi8(x3, x10); + x8 = x1; + x9 = x2; + /* Rounds: 0-3 */ + x0 = x3; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 4-7 */ + x4 = _mm_shuffle_epi8(x4, x10); + x0 = x4; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 8-11 */ + x5 = _mm_shuffle_epi8(x5, x10); + x0 = x5; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 12-15 */ + x6 = _mm_shuffle_epi8(x6, x10); + x0 = x6; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x6; + x7 = _mm_alignr_epi8(x7, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 16-19 */ + x0 = x3; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x3; + x7 = _mm_alignr_epi8(x7, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 20-23 */ + x0 = x4; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x4; + x7 = _mm_alignr_epi8(x7, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 24-27 */ + x0 = x5; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x5; + x7 = _mm_alignr_epi8(x7, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 28-31 */ + x0 = x6; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x6; + x7 = _mm_alignr_epi8(x7, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 32-35 */ + x0 = x3; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x3; + x7 = _mm_alignr_epi8(x7, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 36-39 */ + x0 = x4; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x4; + x7 = _mm_alignr_epi8(x7, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 40-43 */ + x0 = x5; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 160))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x5; + x7 = _mm_alignr_epi8(x7, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 44-47 */ + x0 = x6; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 176))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x6; + x7 = _mm_alignr_epi8(x7, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 48-51 */ + x0 = x3; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 192))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x3; + x7 = _mm_alignr_epi8(x7, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 52-63 */ + x0 = x4; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 208))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x4; + x7 = _mm_alignr_epi8(x7, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x0 = x5; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 224))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x5; + x7 = _mm_alignr_epi8(x7, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x0 = x6; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 240))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x1 = _mm_add_epi32(x1, x8); + x2 = _mm_add_epi32(x2, x9); + x1 = _mm_shuffle_epi32(x1, 0x1b); + x2 = _mm_shuffle_epi32(x2, 0x1b); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 0), x1); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 8), x2); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 16), _mm_unpackhi_epi64(x1, x1)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 24), _mm_unpackhi_epi64(x2, x2)); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("sse2,ssse3,sha") +WOLFSSL_LOCAL int Transform_Sha256_SSE2_Sha_Len(wc_Sha256* sha256, + const byte* data, word32 len) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0 = _mm_setzero_si128(), x1, x2, x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), x10; + word32 zf1; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + rdx = (word64)(word32)len; + + rax = (word64)((word64)(size_t)L_sse2_sha256_sha_k); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(L_sse2_sha256_shuf_mask, 0)); + x1 = _mm_loadl_epi64((const __m128i*)WC_PR(rdi, 0)); + x2 = _mm_loadl_epi64((const __m128i*)WC_PR(rdi, 8)); + x1 = _mm_unpacklo_epi64(x1, _mm_loadl_epi64((const __m128i*)WC_PR(rdi, + 16))); + x2 = _mm_unpacklo_epi64(x2, _mm_loadl_epi64((const __m128i*)WC_PR(rdi, + 24))); + x1 = _mm_shuffle_epi32(x1, 0x1b); + x2 = _mm_shuffle_epi32(x2, 0x1b); + /* Start of loop processing a block */ +L_sha256_sha_len_sse2_start: + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x3 = _mm_shuffle_epi8(x3, x10); + x8 = x1; + x9 = x2; + /* Rounds: 0-3 */ + x0 = x3; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 0))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 4-7 */ + x4 = _mm_shuffle_epi8(x4, x10); + x0 = x4; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 16))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 8-11 */ + x5 = _mm_shuffle_epi8(x5, x10); + x0 = x5; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 32))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 12-15 */ + x6 = _mm_shuffle_epi8(x6, x10); + x0 = x6; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 48))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x6; + x7 = _mm_alignr_epi8(x7, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 16-19 */ + x0 = x3; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 64))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x3; + x7 = _mm_alignr_epi8(x7, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 20-23 */ + x0 = x4; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 80))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x4; + x7 = _mm_alignr_epi8(x7, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 24-27 */ + x0 = x5; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 96))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x5; + x7 = _mm_alignr_epi8(x7, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 28-31 */ + x0 = x6; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 112))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x6; + x7 = _mm_alignr_epi8(x7, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 32-35 */ + x0 = x3; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 128))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x3; + x7 = _mm_alignr_epi8(x7, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 36-39 */ + x0 = x4; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 144))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x4; + x7 = _mm_alignr_epi8(x7, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 40-43 */ + x0 = x5; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 160))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x5; + x7 = _mm_alignr_epi8(x7, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 44-47 */ + x0 = x6; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 176))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x6; + x7 = _mm_alignr_epi8(x7, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 48-51 */ + x0 = x3; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 192))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x3; + x7 = _mm_alignr_epi8(x7, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 52-63 */ + x0 = x4; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 208))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x4; + x7 = _mm_alignr_epi8(x7, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x0 = x5; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 224))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = x5; + x7 = _mm_alignr_epi8(x7, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x0 = x6; + x0 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rax, 240))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + rsi = (word64)(rsi + 0x40); + rdx = (word32)((word32)rdx - 0x40); + zf1 = (word32)rdx; + x1 = _mm_add_epi32(x1, x8); + x2 = _mm_add_epi32(x2, x9); + if ((zf1) != (0)) { + goto L_sha256_sha_len_sse2_start; + } + x1 = _mm_shuffle_epi32(x1, 0x1b); + x2 = _mm_shuffle_epi32(x2, 0x1b); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 0), x1); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 8), x2); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 16), _mm_unpackhi_epi64(x1, x1)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 24), _mm_unpackhi_epi64(x2, x2)); + rax = (word64)(0); + return (int)(word32)rax; +} + +#ifdef HAVE_INTEL_AVX1 +XALIGNED(16) static const word32 L_avx1_sha256_k[] WC_X64I_UNUSED = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +XALIGNED(16) static const word64 L_avx1_sha256_shuf_00BA[] WC_X64I_UNUSED = { + 0x0b0a090803020100ULL, 0xffffffffffffffffULL, +}; + +XALIGNED(16) static const word64 L_avx1_sha256_shuf_DC00[] WC_X64I_UNUSED = { + 0xffffffffffffffffULL, 0x0b0a090803020100ULL, +}; + +XALIGNED(16) static const word64 L_avx1_sha256_flip_mask[] WC_X64I_UNUSED = { + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, +}; + +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL int Transform_Sha256_AVX1(wc_Sha256* sha256, const byte* data) +{ + word64 rdi, rsi, rsp, rbp, r8, r9, r10, r11, r12, r13, r14, r15, rbx, rdx, + rax, rcx; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x11, x12, x13; + XALIGNED(32) WC_X64I_SLOT stk[8]; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + rbp = (word64)((word64)(size_t)L_avx1_sha256_k); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha256_flip_mask, 0)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha256_shuf_00BA, 0)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha256_shuf_DC00, 0)); + r8 = (word32)(WC_L32(rdi, 0)); + r9 = (word32)(WC_L32(rdi, 4)); + r10 = (word32)(WC_L32(rdi, 8)); + r11 = (word32)(WC_L32(rdi, 12)); + r12 = (word32)(WC_L32(rdi, 16)); + r13 = (word32)(WC_L32(rdi, 20)); + r14 = (word32)(WC_L32(rdi, 24)); + r15 = (word32)(WC_L32(rdi, 28)); + /* X0, X1, X2, X3 = W[0..15] */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_shuffle_epi8(x0, x13); + x1 = _mm_shuffle_epi8(x1, x13); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x2 = _mm_shuffle_epi8(x2, x13); + x3 = _mm_shuffle_epi8(x3, x13); + rbx = (word32)((word32)r9); + rdx = (word32)((word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r10); + /* set_w_k_xfer_4: 0 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 0))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 32))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x1, x0, 4); + x4 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x2, x1, 4); + x4 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x3, x2, 4); + x4 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x0, x3, 4); + x4 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 4 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 64))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 96))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x1, x0, 4); + x4 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x2, x1, 4); + x4 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x3, x2, 4); + x4 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x0, x3, 4); + x4 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 8 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 128))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 144))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 160))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 176))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x1, x0, 4); + x4 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x2, x1, 4); + x4 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x3, x2, 4); + x4 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x0, x3, 4); + x4 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 12 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 192))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 208))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 224))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 240))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* rnd_all_4: 0-3 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 1-4 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 2-5 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 3-6 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + WC_S32(rdi, 0) = (word32)(WC_L32(rdi, 0) + (word32)r8); + WC_S32(rdi, 4) = (word32)(WC_L32(rdi, 4) + (word32)r9); + WC_S32(rdi, 8) = (word32)(WC_L32(rdi, 8) + (word32)r10); + WC_S32(rdi, 12) = (word32)(WC_L32(rdi, 12) + (word32)r11); + WC_S32(rdi, 16) = (word32)(WC_L32(rdi, 16) + (word32)r12); + WC_S32(rdi, 20) = (word32)(WC_L32(rdi, 20) + (word32)r13); + WC_S32(rdi, 24) = (word32)(WC_L32(rdi, 24) + (word32)r14); + WC_S32(rdi, 28) = (word32)(WC_L32(rdi, 28) + (word32)r15); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL int Transform_Sha256_AVX1_Len(wc_Sha256* sha256, const byte* data, + word32 len) +{ + word64 rdi, rsi, rbp, rsp, r8, r9, r10, r11, r12, r13, r14, r15, rbx = 0, + rdx = 0, rax = 0, rcx = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), x11, x12, x13; + XALIGNED(32) WC_X64I_SLOT stk[12]; + word32 zf1; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + rbp = (word64)(word32)len; + + rsp = (word64)(size_t)stk + 96; + rsp = (word64)(rsp - 68); + WC_S32(rsp, 64) = (word32)((word32)rbp); + rbp = (word64)((word64)(size_t)L_avx1_sha256_k); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha256_flip_mask, 0)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha256_shuf_00BA, 0)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha256_shuf_DC00, 0)); + r8 = (word32)(WC_L32(rdi, 0)); + r9 = (word32)(WC_L32(rdi, 4)); + r10 = (word32)(WC_L32(rdi, 8)); + r11 = (word32)(WC_L32(rdi, 12)); + r12 = (word32)(WC_L32(rdi, 16)); + r13 = (word32)(WC_L32(rdi, 20)); + r14 = (word32)(WC_L32(rdi, 24)); + r15 = (word32)(WC_L32(rdi, 28)); + /* Start of loop processing a block */ +L_sha256_len_avx1_start: + /* X0, X1, X2, X3 = W[0..15] */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_shuffle_epi8(x0, x13); + x1 = _mm_shuffle_epi8(x1, x13); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x2 = _mm_shuffle_epi8(x2, x13); + x3 = _mm_shuffle_epi8(x3, x13); + rbx = (word32)((word32)r9); + rdx = (word32)((word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r10); + /* set_w_k_xfer_4: 0 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 0))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 32))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x1, x0, 4); + x4 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x2, x1, 4); + x4 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x3, x2, 4); + x4 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x0, x3, 4); + x4 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 4 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 64))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 96))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x1, x0, 4); + x4 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x2, x1, 4); + x4 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x3, x2, 4); + x4 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x0, x3, 4); + x4 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 8 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 128))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 144))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 160))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 176))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x1, x0, 4); + x4 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x2, x1, 4); + x4 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x3, x2, 4); + x4 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rcx = (word32)((word32)rcx ^ (word32)r13); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rcx = (word32)((word32)rcx ^ (word32)r11); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x5 = _mm_alignr_epi8(x0, x3, 4); + x4 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + x8 = _mm_srli_epi32(x5, 18); + x9 = _mm_slli_epi32(x5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + x6 = _mm_or_si128(x7, x6); + x8 = _mm_or_si128(x9, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rcx = (word32)((word32)rcx ^ (word32)r9); + x9 = _mm_srli_epi32(x5, 3); + x6 = _mm_xor_si128(x8, x6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + x5 = _mm_xor_si128(x9, x6); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + x8 = _mm_srli_epi32(x6, 10); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + x6 = _mm_xor_si128(x7, x6); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rcx = (word32)((word32)rcx ^ (word32)r15); + x8 = _mm_srli_epi64(x6, 17); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + x9 = _mm_srli_epi32(x6, 10); + x8 = _mm_xor_si128(x7, x8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + x9 = _mm_xor_si128(x8, x9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 12 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 192))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 208))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 224))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 240))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* rnd_all_4: 0-3 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 1-4 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 2-5 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 3-6 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rdi, 0)); + r9 = (word32)((word32)r9 + WC_L32(rdi, 4)); + r10 = (word32)((word32)r10 + WC_L32(rdi, 8)); + r11 = (word32)((word32)r11 + WC_L32(rdi, 12)); + r12 = (word32)((word32)r12 + WC_L32(rdi, 16)); + r13 = (word32)((word32)r13 + WC_L32(rdi, 20)); + r14 = (word32)((word32)r14 + WC_L32(rdi, 24)); + r15 = (word32)((word32)r15 + WC_L32(rdi, 28)); + rsi = (word64)(rsi + 0x40); + WC_S32(rsp, 64) = (word32)(WC_L32(rsp, 64) - 0x40); + zf1 = (word32)WC_S32(rsp, 64); + WC_S32(rdi, 0) = (word32)((word32)r8); + WC_S32(rdi, 4) = (word32)((word32)r9); + WC_S32(rdi, 8) = (word32)((word32)r10); + WC_S32(rdi, 12) = (word32)((word32)r11); + WC_S32(rdi, 16) = (word32)((word32)r12); + WC_S32(rdi, 20) = (word32)((word32)r13); + WC_S32(rdi, 24) = (word32)((word32)r14); + WC_S32(rdi, 28) = (word32)((word32)r15); + if ((zf1) != (0)) { + goto L_sha256_len_avx1_start; + } + rax = (word64)(0); + return (int)(word32)rax; +} + +XALIGNED(16) static const word32 L_avx1_rorx_sha256_k[] WC_X64I_UNUSED = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +XALIGNED(16) static const word64 L_avx1_rorx_sha256_shuf_00BA[] + WC_X64I_UNUSED = { + 0x0b0a090803020100ULL, 0xffffffffffffffffULL, +}; + +XALIGNED(16) static const word64 L_avx1_rorx_sha256_shuf_DC00[] + WC_X64I_UNUSED = { + 0xffffffffffffffffULL, 0x0b0a090803020100ULL, +}; + +XALIGNED(16) static const word64 L_avx1_rorx_sha256_flip_mask[] + WC_X64I_UNUSED = { + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, +}; + +WC_X64I_TARGET("avx,bmi2") +WOLFSSL_LOCAL int Transform_Sha256_AVX1_RORX(wc_Sha256* sha256, + const byte* data) +{ + word64 rdi, rsi, rsp, rbp, r8, r9, r10, r11, r12, r13, r14, r15, rbx, rdx, + rax, rcx; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x11, x12, x13; + XALIGNED(32) WC_X64I_SLOT stk[8]; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 64); + rbp = (word64)((word64)(size_t)L_avx1_rorx_sha256_k); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_rorx_sha256_flip_mask, + 0)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_rorx_sha256_shuf_00BA, + 0)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_rorx_sha256_shuf_DC00, + 0)); + /* X0, X1, X2, X3 = W[0..15] */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_shuffle_epi8(x0, x13); + x1 = _mm_shuffle_epi8(x1, x13); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x2 = _mm_shuffle_epi8(x2, x13); + x3 = _mm_shuffle_epi8(x3, x13); + r8 = (word32)(WC_L32(rdi, 0)); + r9 = (word32)(WC_L32(rdi, 4)); + r10 = (word32)(WC_L32(rdi, 8)); + r11 = (word32)(WC_L32(rdi, 12)); + r12 = (word32)(WC_L32(rdi, 16)); + r13 = (word32)(WC_L32(rdi, 20)); + r14 = (word32)(WC_L32(rdi, 24)); + r15 = (word32)(WC_L32(rdi, 28)); + /* set_w_k_xfer_4: 0 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 0))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 32))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + rbx = (word32)((word32)r9); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r10); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + x4 = _mm_alignr_epi8(x3, x2, 4); + x5 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + x4 = _mm_alignr_epi8(x0, x3, 4); + x5 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + x4 = _mm_alignr_epi8(x1, x0, 4); + x5 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + x4 = _mm_alignr_epi8(x2, x1, 4); + x5 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 4 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 64))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 96))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + x4 = _mm_alignr_epi8(x3, x2, 4); + x5 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + x4 = _mm_alignr_epi8(x0, x3, 4); + x5 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + x4 = _mm_alignr_epi8(x1, x0, 4); + x5 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + x4 = _mm_alignr_epi8(x2, x1, 4); + x5 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 8 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 128))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 144))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 160))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 176))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + x4 = _mm_alignr_epi8(x3, x2, 4); + x5 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + x4 = _mm_alignr_epi8(x0, x3, 4); + x5 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + x4 = _mm_alignr_epi8(x1, x0, 4); + x5 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + x4 = _mm_alignr_epi8(x2, x1, 4); + x5 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 12 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 192))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 208))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 224))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 240))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + rax = (word32)(0); + /* rnd_all_4: 0-3 */ + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)((word32)r8 + (word32)rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)((word32)r14 + (word32)rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + /* rnd_all_4: 1-4 */ + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)((word32)r12 + (word32)rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)((word32)r10 + (word32)rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + /* rnd_all_4: 2-5 */ + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)((word32)r8 + (word32)rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)((word32)r14 + (word32)rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + /* rnd_all_4: 3-6 */ + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)((word32)r12 + (word32)rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)((word32)r10 + (word32)rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + r8 = (word32)((word32)r8 + (word32)rax); + WC_S32(rdi, 0) = (word32)(WC_L32(rdi, 0) + (word32)r8); + WC_S32(rdi, 4) = (word32)(WC_L32(rdi, 4) + (word32)r9); + WC_S32(rdi, 8) = (word32)(WC_L32(rdi, 8) + (word32)r10); + WC_S32(rdi, 12) = (word32)(WC_L32(rdi, 12) + (word32)r11); + WC_S32(rdi, 16) = (word32)(WC_L32(rdi, 16) + (word32)r12); + WC_S32(rdi, 20) = (word32)(WC_L32(rdi, 20) + (word32)r13); + WC_S32(rdi, 24) = (word32)(WC_L32(rdi, 24) + (word32)r14); + WC_S32(rdi, 28) = (word32)(WC_L32(rdi, 28) + (word32)r15); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx,bmi2") +WOLFSSL_LOCAL int Transform_Sha256_AVX1_RORX_Len(wc_Sha256* sha256, + const byte* data, word32 len) +{ + word64 rdi, rsi, rbp, rsp, r8, r9, r10, r11, r12, r13, r14, r15, rbx = 0, + rdx = 0, rax = 0, rcx = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), x11, x12, x13; + XALIGNED(32) WC_X64I_SLOT stk[12]; + word32 zf1; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + rbp = (word64)(word32)len; + + rsp = (word64)(size_t)stk + 96; + rsp = (word64)(rsp - 68); + WC_S32(rsp, 64) = (word32)((word32)rbp); + rbp = (word64)((word64)(size_t)L_avx1_rorx_sha256_k); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_rorx_sha256_flip_mask, + 0)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_rorx_sha256_shuf_00BA, + 0)); + x12 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_rorx_sha256_shuf_DC00, + 0)); + r8 = (word32)(WC_L32(rdi, 0)); + r9 = (word32)(WC_L32(rdi, 4)); + r10 = (word32)(WC_L32(rdi, 8)); + r11 = (word32)(WC_L32(rdi, 12)); + r12 = (word32)(WC_L32(rdi, 16)); + r13 = (word32)(WC_L32(rdi, 20)); + r14 = (word32)(WC_L32(rdi, 24)); + r15 = (word32)(WC_L32(rdi, 28)); + /* Start of loop processing a block */ +L_sha256_len_avx1_len_rorx_start: + /* X0, X1, X2, X3 = W[0..15] */ + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_shuffle_epi8(x0, x13); + x1 = _mm_shuffle_epi8(x1, x13); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x2 = _mm_shuffle_epi8(x2, x13); + x3 = _mm_shuffle_epi8(x3, x13); + /* set_w_k_xfer_4: 0 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 0))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 32))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + rbx = (word32)((word32)r9); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r10); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + x4 = _mm_alignr_epi8(x3, x2, 4); + x5 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + x4 = _mm_alignr_epi8(x0, x3, 4); + x5 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + x4 = _mm_alignr_epi8(x1, x0, 4); + x5 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + x4 = _mm_alignr_epi8(x2, x1, 4); + x5 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 4 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 64))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 96))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + x4 = _mm_alignr_epi8(x3, x2, 4); + x5 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + x4 = _mm_alignr_epi8(x0, x3, 4); + x5 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + x4 = _mm_alignr_epi8(x1, x0, 4); + x5 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + x4 = _mm_alignr_epi8(x2, x1, 4); + x5 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 8 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 128))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 144))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 160))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 176))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + x4 = _mm_alignr_epi8(x3, x2, 4); + x5 = _mm_alignr_epi8(x1, x0, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x3, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x0); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x0 = _mm_add_epi32(x9, x4); + /* msg_sched done: 0-3 */ + /* msg_sched: 4-7 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + x4 = _mm_alignr_epi8(x0, x3, 4); + x5 = _mm_alignr_epi8(x2, x1, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x0, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x1); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x1 = _mm_add_epi32(x9, x4); + /* msg_sched done: 4-7 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + x4 = _mm_alignr_epi8(x1, x0, 4); + x5 = _mm_alignr_epi8(x3, x2, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + x6 = _mm_shuffle_epi32(x1, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x2); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + x2 = _mm_add_epi32(x9, x4); + /* msg_sched done: 8-11 */ + /* msg_sched: 12-15 */ + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + x4 = _mm_alignr_epi8(x2, x1, 4); + x5 = _mm_alignr_epi8(x0, x3, 4); + /* rnd_0: 1 - 2 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + x6 = _mm_srli_epi32(x5, 7); + x7 = _mm_slli_epi32(x5, 25); + /* rnd_0: 3 - 4 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + x8 = _mm_srli_epi32(x5, 3); + x7 = _mm_or_si128(x7, x6); + /* rnd_0: 5 - 7 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + x6 = _mm_srli_epi32(x5, 18); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + x5 = _mm_slli_epi32(x5, 14); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + x7 = _mm_xor_si128(x7, x5); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + x7 = _mm_xor_si128(x7, x6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + x6 = _mm_shuffle_epi32(x2, 0xfa); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + x5 = _mm_xor_si128(x7, x8); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x8 = _mm_srli_epi32(x6, 10); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + x6 = _mm_srli_epi64(x6, 17); + x4 = _mm_add_epi32(x4, x3); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + x4 = _mm_add_epi32(x4, x5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + x6 = _mm_xor_si128(x6, x7); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + x8 = _mm_xor_si128(x8, x6); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + x8 = _mm_shuffle_epi8(x8, x11); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + x4 = _mm_add_epi32(x4, x8); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + x6 = _mm_shuffle_epi32(x4, 0x50); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + x9 = _mm_srli_epi32(x6, 10); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + x7 = _mm_srli_epi64(x6, 19); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + x6 = _mm_srli_epi64(x6, 17); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + x6 = _mm_xor_si128(x6, x7); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + x9 = _mm_xor_si128(x9, x6); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + x9 = _mm_shuffle_epi8(x9, x12); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + x3 = _mm_add_epi32(x9, x4); + /* msg_sched done: 12-15 */ + /* set_w_k_xfer_4: 12 */ + x4 = _mm_add_epi32(x0, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 192))); + x5 = _mm_add_epi32(x1, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 208))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x4); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x5); + x6 = _mm_add_epi32(x2, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 224))); + x7 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rbp, 240))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x6); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x7); + rax = (word32)(0); + rcx = (word32)(0); + /* rnd_all_4: 0-3 */ + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)((word32)r8 + (word32)rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)((word32)r14 + (word32)rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + /* rnd_all_4: 1-4 */ + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)((word32)r12 + (word32)rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 16)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 20)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)((word32)r10 + (word32)rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 24)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 28)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + /* rnd_all_4: 2-5 */ + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)((word32)r8 + (word32)rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 32)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 36)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)((word32)r14 + (word32)rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 40)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 44)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + /* rnd_all_4: 3-6 */ + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)((word32)r12 + (word32)rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)((word32)r10 + (word32)rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + r8 = (word32)((word32)r8 + (word32)rax); + r8 = (word32)((word32)r8 + WC_L32(rdi, 0)); + r9 = (word32)((word32)r9 + WC_L32(rdi, 4)); + r10 = (word32)((word32)r10 + WC_L32(rdi, 8)); + r11 = (word32)((word32)r11 + WC_L32(rdi, 12)); + r12 = (word32)((word32)r12 + WC_L32(rdi, 16)); + r13 = (word32)((word32)r13 + WC_L32(rdi, 20)); + r14 = (word32)((word32)r14 + WC_L32(rdi, 24)); + r15 = (word32)((word32)r15 + WC_L32(rdi, 28)); + rsi = (word64)(rsi + 0x40); + WC_S32(rsp, 64) = (word32)(WC_L32(rsp, 64) - 0x40); + zf1 = (word32)WC_S32(rsp, 64); + WC_S32(rdi, 0) = (word32)((word32)r8); + WC_S32(rdi, 4) = (word32)((word32)r9); + WC_S32(rdi, 8) = (word32)((word32)r10); + WC_S32(rdi, 12) = (word32)((word32)r11); + WC_S32(rdi, 16) = (word32)((word32)r12); + WC_S32(rdi, 20) = (word32)((word32)r13); + WC_S32(rdi, 24) = (word32)((word32)r14); + WC_S32(rdi, 28) = (word32)((word32)r15); + if ((zf1) != (0)) { + goto L_sha256_len_avx1_len_rorx_start; + } + rax = (word64)(0); + return (int)(word32)rax; +} + +XALIGNED(16) static const word32 L_avx1_sha256_sha_k[] WC_X64I_UNUSED = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +XALIGNED(16) static const word64 L_avx1_sha256_shuf_mask[] WC_X64I_UNUSED = { + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, +}; + +WC_X64I_TARGET("sha,avx") +WOLFSSL_LOCAL int Transform_Sha256_AVX1_Sha(wc_Sha256* sha256, const byte* data) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + + rdx = (word64)((word64)(size_t)L_avx1_sha256_sha_k); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha256_shuf_mask, 0)); + x1 = _mm_loadl_epi64((const __m128i*)WC_PR(rdi, 0)); + x2 = _mm_loadl_epi64((const __m128i*)WC_PR(rdi, 8)); + x1 = _mm_unpacklo_epi64(x1, _mm_loadl_epi64((const __m128i*)WC_PR(rdi, + 16))); + x2 = _mm_unpacklo_epi64(x2, _mm_loadl_epi64((const __m128i*)WC_PR(rdi, + 24))); + x1 = _mm_shuffle_epi32(x1, 0x1b); + x2 = _mm_shuffle_epi32(x2, 0x1b); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x3 = _mm_shuffle_epi8(x3, x10); + x8 = x1; + x9 = x2; + /* Rounds: 0-3 */ + x0 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 4-7 */ + x4 = _mm_shuffle_epi8(x4, x10); + x0 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 8-11 */ + x5 = _mm_shuffle_epi8(x5, x10); + x0 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 12-15 */ + x6 = _mm_shuffle_epi8(x6, x10); + x0 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x6, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 16-19 */ + x0 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x3, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 20-23 */ + x0 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x4, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 24-27 */ + x0 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x5, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 28-31 */ + x0 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x6, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 32-35 */ + x0 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x3, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 36-39 */ + x0 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x4, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 40-43 */ + x0 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 160))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x5, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 44-47 */ + x0 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 176))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x6, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 48-51 */ + x0 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 192))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x3, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 52-63 */ + x0 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 208))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x4, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x0 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 224))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x5, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x0 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 240))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x1 = _mm_add_epi32(x1, x8); + x2 = _mm_add_epi32(x2, x9); + x1 = _mm_shuffle_epi32(x1, 0x1b); + x2 = _mm_shuffle_epi32(x2, 0x1b); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 0), x1); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 8), x2); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 16), _mm_unpackhi_epi64(x1, x1)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 24), _mm_unpackhi_epi64(x2, x2)); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("sha,avx") +WOLFSSL_LOCAL int Transform_Sha256_AVX1_Sha_Len(wc_Sha256* sha256, + const byte* data, word32 len) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0 = _mm_setzero_si128(), x1, x2, x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), x10; + word32 zf1; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + rdx = (word64)(word32)len; + + rax = (word64)((word64)(size_t)L_avx1_sha256_sha_k); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha256_shuf_mask, 0)); + x1 = _mm_loadl_epi64((const __m128i*)WC_PR(rdi, 0)); + x2 = _mm_loadl_epi64((const __m128i*)WC_PR(rdi, 8)); + x1 = _mm_unpacklo_epi64(x1, _mm_loadl_epi64((const __m128i*)WC_PR(rdi, + 16))); + x2 = _mm_unpacklo_epi64(x2, _mm_loadl_epi64((const __m128i*)WC_PR(rdi, + 24))); + x1 = _mm_shuffle_epi32(x1, 0x1b); + x2 = _mm_shuffle_epi32(x2, 0x1b); + /* Start of loop processing a block */ +L_sha256_sha_len_avx1_start: + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x3 = _mm_shuffle_epi8(x3, x10); + x8 = x1; + x9 = x2; + /* Rounds: 0-3 */ + x0 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rax, 0))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 4-7 */ + x4 = _mm_shuffle_epi8(x4, x10); + x0 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rax, 16))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 8-11 */ + x5 = _mm_shuffle_epi8(x5, x10); + x0 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(rax, 32))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 12-15 */ + x6 = _mm_shuffle_epi8(x6, x10); + x0 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(rax, 48))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x6, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 16-19 */ + x0 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rax, 64))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x3, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 20-23 */ + x0 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rax, 80))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x4, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 24-27 */ + x0 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(rax, 96))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x5, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 28-31 */ + x0 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(rax, 112))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x6, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 32-35 */ + x0 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rax, 128))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x3, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 36-39 */ + x0 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rax, 144))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x4, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x3 = _mm_sha256msg1_epu32(x3, x4); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 40-43 */ + x0 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(rax, 160))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x5, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x4 = _mm_sha256msg1_epu32(x4, x5); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 44-47 */ + x0 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(rax, 176))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x6, x5, 4); + x3 = _mm_add_epi32(x3, x7); + x3 = _mm_sha256msg2_epu32(x3, x6); + x0 = _mm_shuffle_epi32(x0, 0xe); + x5 = _mm_sha256msg1_epu32(x5, x6); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 48-51 */ + x0 = _mm_add_epi32(x3, _mm_loadu_si128((const __m128i*)WC_PR(rax, 192))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x3, x6, 4); + x4 = _mm_add_epi32(x4, x7); + x4 = _mm_sha256msg2_epu32(x4, x3); + x0 = _mm_shuffle_epi32(x0, 0xe); + x6 = _mm_sha256msg1_epu32(x6, x3); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + /* Rounds: 52-63 */ + x0 = _mm_add_epi32(x4, _mm_loadu_si128((const __m128i*)WC_PR(rax, 208))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x4, x3, 4); + x5 = _mm_add_epi32(x5, x7); + x5 = _mm_sha256msg2_epu32(x5, x4); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x0 = _mm_add_epi32(x5, _mm_loadu_si128((const __m128i*)WC_PR(rax, 224))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x7 = _mm_alignr_epi8(x5, x4, 4); + x6 = _mm_add_epi32(x6, x7); + x6 = _mm_sha256msg2_epu32(x6, x5); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + x0 = _mm_add_epi32(x6, _mm_loadu_si128((const __m128i*)WC_PR(rax, 240))); + x2 = _mm_sha256rnds2_epu32(x2, x1, x0); + x0 = _mm_shuffle_epi32(x0, 0xe); + x1 = _mm_sha256rnds2_epu32(x1, x2, x0); + rsi = (word64)(rsi + 0x40); + rdx = (word32)((word32)rdx - 0x40); + zf1 = (word32)rdx; + x1 = _mm_add_epi32(x1, x8); + x2 = _mm_add_epi32(x2, x9); + if ((zf1) != (0)) { + goto L_sha256_sha_len_avx1_start; + } + x1 = _mm_shuffle_epi32(x1, 0x1b); + x2 = _mm_shuffle_epi32(x2, 0x1b); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 0), x1); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 8), x2); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 16), _mm_unpackhi_epi64(x1, x1)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 24), _mm_unpackhi_epi64(x2, x2)); + rax = (word64)(0); + return (int)(word32)rax; +} + +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +XALIGNED(16) static const word32 L_avx2_sha256_k[] WC_X64I_UNUSED = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +XALIGNED(32) static const word64 L_avx2_sha256_shuf_00BA[] WC_X64I_UNUSED = { + 0x0b0a090803020100ULL, 0xffffffffffffffffULL, + 0x0b0a090803020100ULL, 0xffffffffffffffffULL, +}; + +XALIGNED(32) static const word64 L_avx2_sha256_shuf_DC00[] WC_X64I_UNUSED = { + 0xffffffffffffffffULL, 0x0b0a090803020100ULL, + 0xffffffffffffffffULL, 0x0b0a090803020100ULL, +}; + +XALIGNED(32) static const word64 L_avx2_sha256_flip_mask[] WC_X64I_UNUSED = { + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL int Transform_Sha256_AVX2(wc_Sha256* sha256, const byte* data) +{ + word64 rdi, rsi, rsp, rbp, r8, r9, r10, r11, r12, r13, r14, r15, rbx, rdx, + rax, rcx; + __m128i x13; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y11, y12; + XALIGNED(32) WC_X64I_SLOT stk[64]; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + + rsp = (word64)(size_t)stk + 512; + rsp = (word64)(rsp - 512); + rbp = (word64)((word64)(size_t)L_avx2_sha256_k); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_sha256_flip_mask, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_sha256_shuf_00BA, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_sha256_shuf_DC00, 0)); + r8 = (word32)(WC_L32(rdi, 0)); + r9 = (word32)(WC_L32(rdi, 4)); + r10 = (word32)(WC_L32(rdi, 8)); + r11 = (word32)(WC_L32(rdi, 12)); + r12 = (word32)(WC_L32(rdi, 16)); + r13 = (word32)(WC_L32(rdi, 20)); + r14 = (word32)(WC_L32(rdi, 24)); + r15 = (word32)(WC_L32(rdi, 28)); + /* X0, X1, X2, X3 = W[0..15] */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 16))); + y0 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y0), + x13)); + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y1), + x13)); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 32))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 48))); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y2), + x13)); + y3 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y3), + x13)); + rbx = (word32)((word32)r9); + rdx = (word32)((word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r10); + /* set_w_k_xfer_4: 0 */ + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 32), y5); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 64), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 96), y5); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y0 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 0-3 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 32)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 36)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 40)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 44)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y1 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 8-11 */ + /* msg_sched: 16-19 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 64)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 68)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 72)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 76)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y2 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 16-19 */ + /* msg_sched: 24-27 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 96)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 100)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 104)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 108)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y3 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 24-27 */ + /* set_w_k_xfer_4: 4 */ + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 160), y5); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 192), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 224), y5); + /* msg_sched: 32-35 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 128)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 132)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 136)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 140)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y0 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 32-35 */ + /* msg_sched: 40-43 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 160)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 164)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 168)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 172)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y1 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 40-43 */ + /* msg_sched: 48-51 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 192)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 196)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 200)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 204)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y2 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 48-51 */ + /* msg_sched: 56-59 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 224)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 228)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 232)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 236)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y3 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 56-59 */ + /* set_w_k_xfer_4: 8 */ + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y5); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y5); + /* msg_sched: 64-67 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 256)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 260)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 264)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 268)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y0 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 64-67 */ + /* msg_sched: 72-75 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 288)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 292)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 296)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 300)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y1 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 72-75 */ + /* msg_sched: 80-83 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 320)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 324)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 328)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 332)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y2 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 80-83 */ + /* msg_sched: 88-91 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 352)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 356)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 360)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 364)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y3 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 88-91 */ + /* set_w_k_xfer_4: 12 */ + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y5); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y5); + /* rnd_all_4: 24-27 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 384)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 388)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 392)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 396)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 26-29 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 416)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 420)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 424)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 428)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 28-31 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 448)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 452)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 456)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 460)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 30-33 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 480)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 484)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 488)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 492)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + WC_S32(rdi, 0) = (word32)(WC_L32(rdi, 0) + (word32)r8); + WC_S32(rdi, 4) = (word32)(WC_L32(rdi, 4) + (word32)r9); + WC_S32(rdi, 8) = (word32)(WC_L32(rdi, 8) + (word32)r10); + WC_S32(rdi, 12) = (word32)(WC_L32(rdi, 12) + (word32)r11); + WC_S32(rdi, 16) = (word32)(WC_L32(rdi, 16) + (word32)r12); + WC_S32(rdi, 20) = (word32)(WC_L32(rdi, 20) + (word32)r13); + WC_S32(rdi, 24) = (word32)(WC_L32(rdi, 24) + (word32)r14); + WC_S32(rdi, 28) = (word32)(WC_L32(rdi, 28) + (word32)r15); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL int Transform_Sha256_AVX2_Len(wc_Sha256* sha256, const byte* data, + word32 len) +{ + word64 rdi, rsi, rbp, rsp, r8 = 0, r9 = 0, r10 = 0, r11 = 0, r12 = 0, + r13 = 0, r14 = 0, r15 = 0, rbx = 0, rdx = 0, rax = 0, rcx = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[68]; + byte zf1; + word32 zf2; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + rbp = (word64)(word32)len; + + rsp = (word64)(size_t)stk + 544; + rsp = (word64)(rsp - 516); + zf1 = ((byte)rbp & 0x40); + WC_S32(rsp, 512) = (word32)((word32)rbp); + if ((zf1) == (0)) { + goto L_sha256_len_avx2_block; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + (void)Transform_Sha256_AVX2((wc_Sha256*)(size_t)rdi, (const byte*)( + size_t)rsi); + rsi = (word64)(rsi + 0x40); + WC_S32(rsp, 512) = (word32)(WC_L32(rsp, 512) - 0x40); + if (((word32)WC_S32(rsp, 512)) == (0)) { + goto L_sha256_len_avx2_done; + } +L_sha256_len_avx2_block: + rbp = (word64)((word64)(size_t)L_avx2_sha256_k); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_sha256_flip_mask, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_sha256_shuf_00BA, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_sha256_shuf_DC00, 0)); + r8 = (word32)(WC_L32(rdi, 0)); + r9 = (word32)(WC_L32(rdi, 4)); + r10 = (word32)(WC_L32(rdi, 8)); + r11 = (word32)(WC_L32(rdi, 12)); + r12 = (word32)(WC_L32(rdi, 16)); + r13 = (word32)(WC_L32(rdi, 20)); + r14 = (word32)(WC_L32(rdi, 24)); + r15 = (word32)(WC_L32(rdi, 28)); + /* Start of loop processing two blocks */ +L_sha256_len_avx2_start: + /* X0, X1, X2, X3 = W[0..15] */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 16))); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 64))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 80))); + y0 = _mm256_inserti128_si256(y0, _mm256_castsi256_si128(y4), 1); + y1 = _mm256_inserti128_si256(y1, _mm256_castsi256_si128(y5), 1); + y0 = _mm256_shuffle_epi8(y0, y13); + y1 = _mm256_shuffle_epi8(y1, y13); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 32))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 48))); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 96))); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 112))); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y6), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y7), 1); + y2 = _mm256_shuffle_epi8(y2, y13); + y3 = _mm256_shuffle_epi8(y3, y13); + rbx = (word32)((word32)r9); + rdx = (word32)((word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r10); + /* set_w_k_xfer_4: 0 */ + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 32), y5); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 64), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 96), y5); + /* msg_sched: 0-3 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y0 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 0-3 */ + /* msg_sched: 8-11 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 32)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 36)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 40)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 44)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y1 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 8-11 */ + /* msg_sched: 16-19 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 64)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 68)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 72)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 76)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y2 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 16-19 */ + /* msg_sched: 24-27 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 96)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 100)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 104)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 108)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y3 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 24-27 */ + /* set_w_k_xfer_4: 4 */ + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 160), y5); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 192), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 224), y5); + /* msg_sched: 32-35 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 128)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 132)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 136)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 140)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y0 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 32-35 */ + /* msg_sched: 40-43 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 160)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 164)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 168)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 172)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y1 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 40-43 */ + /* msg_sched: 48-51 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 192)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 196)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 200)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 204)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y2 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 48-51 */ + /* msg_sched: 56-59 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 224)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 228)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 232)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 236)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y3 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 56-59 */ + /* set_w_k_xfer_4: 8 */ + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y5); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y5); + /* msg_sched: 64-67 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 256)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 260)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 264)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 268)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y0 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 64-67 */ + /* msg_sched: 72-75 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 288)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 292)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 296)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 300)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y1 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 72-75 */ + /* msg_sched: 80-83 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r9); + rcx = (word32)((word32)r13); + r15 = (word32)((word32)r15 + WC_L32(rsp, 320)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx & (word32)r12); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r12); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((word32)r8); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)r12); + r14 = (word32)((word32)r14 + WC_L32(rsp, 324)); + rcx = (word32)((word32)rcx ^ (word32)r13); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r11); + r14 = (word32)((word32)r14 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r15); + rcx = (word32)((word32)r11); + r13 = (word32)((word32)r13 + WC_L32(rsp, 328)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r10); + r13 = (word32)((word32)r13 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((word32)r14); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r14); + rcx = (word32)((word32)r10); + r12 = (word32)((word32)r12 + WC_L32(rsp, 332)); + rcx = (word32)((word32)rcx ^ (word32)r11); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r9); + r12 = (word32)((word32)r12 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + y2 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 80-83 */ + /* msg_sched: 88-91 */ + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 2 */ + rax = (word32)((word32)r13); + rcx = (word32)((word32)r9); + r11 = (word32)((word32)r11 + WC_L32(rsp, 352)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx & (word32)r8); + y6 = _mm256_srli_epi32(y5, 7); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 3 - 4 */ + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r8); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((word32)r12); + y8 = _mm256_srli_epi32(y5, 18); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 5 - 6 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + y6 = _mm256_or_si256(y7, y6); + y8 = _mm256_or_si256(y9, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + /* rnd_1: 0 - 1 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)r8); + r10 = (word32)((word32)r10 + WC_L32(rsp, 356)); + rcx = (word32)((word32)rcx ^ (word32)r9); + y9 = _mm256_srli_epi32(y5, 3); + y6 = _mm256_xor_si256(y8, y6); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r15); + r10 = (word32)((word32)r10 + (word32)rcx); + y5 = _mm256_xor_si256(y9, y6); + y6 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_srli_epi32(y6, 10); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 6 - 7 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + /* rnd_0: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_srli_epi64(y6, 17); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 1 - 3 */ + rax = (word32)((word32)r11); + rcx = (word32)((word32)r15); + r9 = (word32)((word32)r9 + WC_L32(rsp, 360)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r14); + r9 = (word32)((word32)r9 + (word32)rcx); + y6 = _mm256_xor_si256(y7, y6); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 4 - 4 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((word32)r10); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 5 - 5 */ + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + /* rnd_1: 0 - 0 */ + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_1: 1 - 1 */ + rbx = (word32)((word32)r10); + rcx = (word32)((word32)r14); + r8 = (word32)((word32)r8 + WC_L32(rsp, 364)); + rcx = (word32)((word32)rcx ^ (word32)r15); + y8 = _mm256_srli_epi64(y6, 17); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 2 - 3 */ + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r13); + r8 = (word32)((word32)r8 + (word32)rcx); + y9 = _mm256_srli_epi32(y6, 10); + y8 = _mm256_xor_si256(y7, y8); + /* rnd_1: 4 - 5 */ + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + y9 = _mm256_xor_si256(y8, y9); + /* rnd_1: 6 - 6 */ + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 7 - 7 */ + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + y3 = _mm256_add_epi32(y9, y4); + /* msg_sched done: 88-91 */ + /* set_w_k_xfer_4: 12 */ + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y5); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y5); + /* rnd_all_4: 24-27 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 384)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 388)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 392)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 396)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 26-29 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 416)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 420)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 424)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 428)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 28-31 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 448)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 452)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 456)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 460)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 30-33 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 480)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 484)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 488)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 492)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rdi, 0)); + r9 = (word32)((word32)r9 + WC_L32(rdi, 4)); + r10 = (word32)((word32)r10 + WC_L32(rdi, 8)); + r11 = (word32)((word32)r11 + WC_L32(rdi, 12)); + r12 = (word32)((word32)r12 + WC_L32(rdi, 16)); + r13 = (word32)((word32)r13 + WC_L32(rdi, 20)); + r14 = (word32)((word32)r14 + WC_L32(rdi, 24)); + r15 = (word32)((word32)r15 + WC_L32(rdi, 28)); + WC_S32(rdi, 0) = (word32)((word32)r8); + WC_S32(rdi, 4) = (word32)((word32)r9); + WC_S32(rdi, 8) = (word32)((word32)r10); + WC_S32(rdi, 12) = (word32)((word32)r11); + WC_S32(rdi, 16) = (word32)((word32)r12); + WC_S32(rdi, 20) = (word32)((word32)r13); + WC_S32(rdi, 24) = (word32)((word32)r14); + WC_S32(rdi, 28) = (word32)((word32)r15); + rbx = (word32)((word32)r9); + rdx = (word32)((word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r10); + /* rnd_all_4: 1-4 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 16)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 20)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 24)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 28)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 3-6 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 5-8 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 80)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 84)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 88)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 92)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 7-10 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 112)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 116)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 120)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 124)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 9-12 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 144)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 148)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 152)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 156)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 11-14 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 176)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 180)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 184)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 188)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 13-16 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 208)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 212)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 216)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 220)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 15-18 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 240)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 244)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 248)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 252)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 17-20 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 272)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 276)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 280)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 284)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 19-22 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 304)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 308)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 312)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 316)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 21-24 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 336)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 340)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 344)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 348)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 23-26 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 368)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 372)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 376)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 380)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 25-28 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 400)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 404)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 408)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 412)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 27-30 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 432)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 436)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 440)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 444)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + /* rnd_all_4: 29-32 */ + r15 = (word32)((word32)r15 + WC_L32(rsp, 464)); + rcx = (word32)((word32)r13); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r15 = (word32)((word32)r15 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r12); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r11 = (word32)((word32)r11 + (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r11); + r15 = (word32)((word32)r15 + (word32)rcx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 468)); + rcx = (word32)((word32)r12); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r14 = (word32)((word32)r14 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r11); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r10 = (word32)((word32)r10 + (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r10); + r14 = (word32)((word32)r14 + (word32)rcx); + r13 = (word32)((word32)r13 + WC_L32(rsp, 472)); + rcx = (word32)((word32)r11); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r13 = (word32)((word32)r13 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r10); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r9 = (word32)((word32)r9 + (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r9); + r13 = (word32)((word32)r13 + (word32)rcx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 476)); + rcx = (word32)((word32)r10); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r12 = (word32)((word32)r12 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r9); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r8 = (word32)((word32)r8 + (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r8); + r12 = (word32)((word32)r12 + (word32)rcx); + /* rnd_all_4: 31-34 */ + r11 = (word32)((word32)r11 + WC_L32(rsp, 496)); + rcx = (word32)((word32)r9); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r11 = (word32)((word32)r11 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r8); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r15 = (word32)((word32)r15 + (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r15); + r11 = (word32)((word32)r11 + (word32)rcx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 500)); + rcx = (word32)((word32)r8); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r10 = (word32)((word32)r10 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r15); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r14 = (word32)((word32)r14 + (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r14); + r10 = (word32)((word32)r10 + (word32)rcx); + r9 = (word32)((word32)r9 + WC_L32(rsp, 504)); + rcx = (word32)((word32)r15); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)r8); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r9 = (word32)((word32)r9 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r14); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rbx = (word32)((word32)rbx & (word32)rax); + rcx = (word32)((word32)rcx ^ (word32)r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r13 = (word32)((word32)r13 + (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rbx); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r13); + r9 = (word32)((word32)r9 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 508)); + rcx = (word32)((word32)r14); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 14) | (((word32)rdx) << 18)); + rcx = (word32)((word32)rcx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)r15); + rdx = (word32)((((word32)rdx) >> 5) | (((word32)rdx) << 27)); + r8 = (word32)((word32)r8 + (word32)rcx); + rdx = (word32)((word32)rdx ^ (word32)r13); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)rdx) >> 6) | (((word32)rdx) << 26)); + rcx = (word32)((word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rcx = (word32)((((word32)rcx) >> 9) | (((word32)rcx) << 23)); + rax = (word32)((word32)rax & (word32)rbx); + rcx = (word32)((word32)rcx ^ (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((((word32)rcx) >> 11) | (((word32)rcx) << 21)); + r12 = (word32)((word32)r12 + (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)((((word32)rcx) >> 2) | (((word32)rcx) << 30)); + rdx = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)rcx); + r8 = (word32)((word32)r8 + WC_L32(rdi, 0)); + r9 = (word32)((word32)r9 + WC_L32(rdi, 4)); + r10 = (word32)((word32)r10 + WC_L32(rdi, 8)); + r11 = (word32)((word32)r11 + WC_L32(rdi, 12)); + r12 = (word32)((word32)r12 + WC_L32(rdi, 16)); + r13 = (word32)((word32)r13 + WC_L32(rdi, 20)); + r14 = (word32)((word32)r14 + WC_L32(rdi, 24)); + r15 = (word32)((word32)r15 + WC_L32(rdi, 28)); + rsi = (word64)(rsi + 0x80); + WC_S32(rsp, 512) = (word32)(WC_L32(rsp, 512) - 0x80); + zf2 = (word32)WC_S32(rsp, 512); + WC_S32(rdi, 0) = (word32)((word32)r8); + WC_S32(rdi, 4) = (word32)((word32)r9); + WC_S32(rdi, 8) = (word32)((word32)r10); + WC_S32(rdi, 12) = (word32)((word32)r11); + WC_S32(rdi, 16) = (word32)((word32)r12); + WC_S32(rdi, 20) = (word32)((word32)r13); + WC_S32(rdi, 24) = (word32)((word32)r14); + WC_S32(rdi, 28) = (word32)((word32)r15); + if ((zf2) != (0)) { + goto L_sha256_len_avx2_start; + } +L_sha256_len_avx2_done: + rax = (word64)(0); + return (int)(word32)rax; +} + +XALIGNED(16) static const word32 L_avx2_rorx_sha256_k[] WC_X64I_UNUSED = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +XALIGNED(32) static const word64 L_avx2_rorx_sha256_flip_mask[] + WC_X64I_UNUSED = { + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, +}; + +XALIGNED(32) static const word64 L_avx2_rorx_sha256_shuf_00BA[] + WC_X64I_UNUSED = { + 0x0b0a090803020100ULL, 0xffffffffffffffffULL, + 0x0b0a090803020100ULL, 0xffffffffffffffffULL, +}; + +XALIGNED(32) static const word64 L_avx2_rorx_sha256_shuf_DC00[] + WC_X64I_UNUSED = { + 0xffffffffffffffffULL, 0x0b0a090803020100ULL, + 0xffffffffffffffffULL, 0x0b0a090803020100ULL, +}; + +WC_X64I_TARGET("avx2,bmi2") +WOLFSSL_LOCAL int Transform_Sha256_AVX2_RORX(wc_Sha256* sha256, + const byte* data) +{ + word64 rdi, rsi, rsp, rbp, r8, r9, r10, r11, r12, r13, r14, r15, rbx, rdx, + rax, rcx; + __m128i x13; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y11, y12; + XALIGNED(32) WC_X64I_SLOT stk[64]; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + + rsp = (word64)(size_t)stk + 512; + rsp = (word64)(rsp - 512); + rbp = (word64)((word64)(size_t)L_avx2_rorx_sha256_k); + x13 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx2_rorx_sha256_flip_mask, + 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_rorx_sha256_shuf_00BA, + 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_rorx_sha256_shuf_DC00, + 0)); + /* X0, X1, X2, X3 = W[0..15] */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 16))); + y0 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y0), + x13)); + y1 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y1), + x13)); + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 32), y5); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 32))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 48))); + y2 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y2), + x13)); + y3 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y3), + x13)); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 64), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 96), y5); + r8 = (word32)(WC_L32(rdi, 0)); + r9 = (word32)(WC_L32(rdi, 4)); + r10 = (word32)(WC_L32(rdi, 8)); + r11 = (word32)(WC_L32(rdi, 12)); + r12 = (word32)(WC_L32(rdi, 16)); + r13 = (word32)(WC_L32(rdi, 20)); + r14 = (word32)(WC_L32(rdi, 24)); + r15 = (word32)(WC_L32(rdi, 28)); + rbx = (word32)((word32)r9); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r10); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y0 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 128), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 32)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 36)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 40)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 44)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y1 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 160), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 64)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 68)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 72)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 76)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y2 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 192), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 96)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 100)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 104)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 108)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y3 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 224), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 128)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 132)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 136)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 140)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y0 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 160)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 164)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 168)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 172)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y1 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 192)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 196)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 200)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 204)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y2 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 224)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 228)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 232)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 236)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y3 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 256)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 260)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 264)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 268)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y0 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 288)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 292)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 296)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 300)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y1 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 320)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 324)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 328)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 332)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y2 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 352)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 356)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 360)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 364)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y3 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y4); + rax = (word32)(0); + rcx = (word32)(0); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 384)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 388)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 392)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 396)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 416)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 420)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 424)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 428)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 448)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 452)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 456)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 460)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 480)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 484)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 488)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 492)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + r8 = (word32)((word32)r8 + (word32)rax); + WC_S32(rdi, 0) = (word32)(WC_L32(rdi, 0) + (word32)r8); + WC_S32(rdi, 4) = (word32)(WC_L32(rdi, 4) + (word32)r9); + WC_S32(rdi, 8) = (word32)(WC_L32(rdi, 8) + (word32)r10); + WC_S32(rdi, 12) = (word32)(WC_L32(rdi, 12) + (word32)r11); + WC_S32(rdi, 16) = (word32)(WC_L32(rdi, 16) + (word32)r12); + WC_S32(rdi, 20) = (word32)(WC_L32(rdi, 20) + (word32)r13); + WC_S32(rdi, 24) = (word32)(WC_L32(rdi, 24) + (word32)r14); + WC_S32(rdi, 28) = (word32)(WC_L32(rdi, 28) + (word32)r15); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx2,bmi2") +WOLFSSL_LOCAL int Transform_Sha256_AVX2_RORX_Len(wc_Sha256* sha256, + const byte* data, word32 len) +{ + word64 rdi, rsi, rbp, rsp, r8 = 0, r9 = 0, r10 = 0, r11 = 0, r12 = 0, + r13 = 0, r14 = 0, r15 = 0, rbx = 0, rdx = 0, rax = 0, rcx = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[68]; + byte zf1; + word32 zf2; + + rdi = (word64)(size_t)sha256; + rsi = (word64)(size_t)data; + rbp = (word64)(word32)len; + + rsp = (word64)(size_t)stk + 544; + rsp = (word64)(rsp - 516); + zf1 = ((byte)rbp & 0x40); + WC_S32(rsp, 512) = (word32)((word32)rbp); + if ((zf1) == (0)) { + goto L_sha256_len_avx2_rorx_block; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + (void)Transform_Sha256_AVX2_RORX((wc_Sha256*)(size_t)rdi, (const byte*)( + size_t)rsi); + rsi = (word64)(rsi + 0x40); + WC_S32(rsp, 512) = (word32)(WC_L32(rsp, 512) - 0x40); + if (((word32)WC_S32(rsp, 512)) == (0)) { + goto L_sha256_len_avx2_rorx_done; + } +L_sha256_len_avx2_rorx_block: + rbp = (word64)((word64)(size_t)L_avx2_rorx_sha256_k); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_rorx_sha256_flip_mask, + 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_rorx_sha256_shuf_00BA, + 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_rorx_sha256_shuf_DC00, + 0)); + r8 = (word32)(WC_L32(rdi, 0)); + r9 = (word32)(WC_L32(rdi, 4)); + r10 = (word32)(WC_L32(rdi, 8)); + r11 = (word32)(WC_L32(rdi, 12)); + r12 = (word32)(WC_L32(rdi, 16)); + r13 = (word32)(WC_L32(rdi, 20)); + r14 = (word32)(WC_L32(rdi, 24)); + r15 = (word32)(WC_L32(rdi, 28)); + /* Start of loop processing two blocks */ +L_sha256_len_avx2_rorx_start: + /* X0, X1, X2, X3 = W[0..15] */ + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 16))); + y0 = _mm256_inserti128_si256(y0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 64)), 1); + y1 = _mm256_inserti128_si256(y1, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 80)), 1); + y0 = _mm256_shuffle_epi8(y0, y13); + y1 = _mm256_shuffle_epi8(y1, y13); + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y5 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 32), y5); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 32))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 48))); + y2 = _mm256_inserti128_si256(y2, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 96)), 1); + y3 = _mm256_inserti128_si256(y3, _mm_loadu_si128((const __m128i*)WC_PR(rsi, + 112)), 1); + y2 = _mm256_shuffle_epi8(y2, y13); + y3 = _mm256_shuffle_epi8(y3, y13); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y5 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 64), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 96), y5); + rbx = (word32)((word32)r9); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rbx = (word32)((word32)rbx ^ (word32)r10); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 0)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 4)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 8)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 12)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y0 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 128), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 32)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 36)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 40)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 44)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y1 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 160), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 64)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 68)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 72)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 76)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y2 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 192), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 96)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 100)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 104)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 108)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y3 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 224), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 128)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 132)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 136)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 140)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y0 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 256), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 160)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 164)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 168)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 172)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y1 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 288), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 192)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 196)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 200)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 204)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y2 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 320), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 224)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 228)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 232)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 236)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y3 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 352), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 256)); + y5 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 260)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y3, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y0); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 264)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 268)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y0 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 384), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 288)); + y5 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 292)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y0, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y1); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 296)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 300)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y1 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 416), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r13); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r15 = (word32)((word32)r15 + WC_L32(rsp, 320)); + y5 = _mm256_alignr_epi8(y3, y2, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + y4 = _mm256_alignr_epi8(y1, y0, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r12); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rax = (word32)((word32)rax ^ (word32)r14); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r8); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + r15 = (word32)((word32)r15 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r12); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r14 = (word32)((word32)r14 + WC_L32(rsp, 324)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r11); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + y7 = _mm256_shuffle_epi32(y1, 0xfa); + /* rnd_1: 3 - 3 */ + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r13); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r10 = (word32)((word32)r10 + (word32)r14); + rbx = (word32)((word32)r8); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + r14 = (word32)((word32)r14 + (word32)rax); + y4 = _mm256_add_epi32(y4, y2); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r11); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r13 = (word32)((word32)r13 + WC_L32(rsp, 328)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r10); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rax = (word32)((word32)rax ^ (word32)r12); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r14); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + r13 = (word32)((word32)r13 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r10); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r12 = (word32)((word32)r12 + WC_L32(rsp, 332)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r9); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r11); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + y2 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r8 = (word32)((word32)r8 + (word32)r12); + rbx = (word32)((word32)r14); + y4 = _mm256_add_epi32(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + r12 = (word32)((word32)r12 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 448), y4); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r9); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r11 = (word32)((word32)r11 + WC_L32(rsp, 352)); + y5 = _mm256_alignr_epi8(y0, y3, 4); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + y4 = _mm256_alignr_epi8(y2, y1, 4); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r8); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + y6 = _mm256_srli_epi32(y5, 7); + /* rnd_0: 3 - 3 */ + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rax = (word32)((word32)rax ^ (word32)r10); + y7 = _mm256_slli_epi32(y5, 25); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + y8 = _mm256_srli_epi32(y5, 18); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + y9 = _mm256_slli_epi32(y5, 14); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r12); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y6 = _mm256_or_si256(y6, y7); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + r11 = (word32)((word32)r11 + (word32)rbx); + y8 = _mm256_or_si256(y8, y9); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r8); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r10 = (word32)((word32)r10 + WC_L32(rsp, 356)); + y9 = _mm256_srli_epi32(y5, 3); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + y6 = _mm256_xor_si256(y6, y8); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r15); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + y7 = _mm256_shuffle_epi32(y2, 0xfa); + /* rnd_1: 3 - 3 */ + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r9); + y5 = _mm256_xor_si256(y9, y6); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + y8 = _mm256_srli_epi32(y7, 10); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r14 = (word32)((word32)r14 + (word32)r10); + rbx = (word32)((word32)r12); + y6 = _mm256_srli_epi64(y7, 19); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + y7 = _mm256_srli_epi64(y7, 17); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + r10 = (word32)((word32)r10 + (word32)rax); + y4 = _mm256_add_epi32(y4, y3); + /* rnd_0: 0 - 0 */ + rax = (word32)((word32)r15); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r9 = (word32)((word32)r9 + WC_L32(rsp, 360)); + y6 = _mm256_xor_si256(y6, y7); + /* rnd_0: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + y8 = _mm256_xor_si256(y8, y6); + /* rnd_0: 2 - 2 */ + rax = (word32)((word32)rax & (word32)r14); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + y4 = _mm256_add_epi32(y4, y5); + /* rnd_0: 3 - 3 */ + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rax = (word32)((word32)rax ^ (word32)r8); + y8 = _mm256_shuffle_epi8(y8, y11); + /* rnd_0: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + y4 = _mm256_add_epi32(y4, y8); + /* rnd_0: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + y6 = _mm256_shuffle_epi32(y4, 0x50); + /* rnd_0: 6 - 6 */ + rax = (word32)((word32)rax ^ (word32)r10); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx & (word32)rax); + y8 = _mm256_srli_epi64(y6, 17); + /* rnd_0: 7 - 7 */ + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + r9 = (word32)((word32)r9 + (word32)rbx); + y7 = _mm256_srli_epi64(y6, 19); + /* rnd_1: 0 - 0 */ + rbx = (word32)((word32)r14); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r8 = (word32)((word32)r8 + WC_L32(rsp, 364)); + y9 = _mm256_srli_epi32(y6, 10); + /* rnd_1: 1 - 1 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + y8 = _mm256_xor_si256(y8, y7); + /* rnd_1: 2 - 2 */ + rbx = (word32)((word32)rbx & (word32)r13); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + y9 = _mm256_xor_si256(y9, y8); + /* rnd_1: 3 - 3 */ + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rbx = (word32)((word32)rbx ^ (word32)r15); + y9 = _mm256_shuffle_epi8(y9, y12); + /* rnd_1: 4 - 4 */ + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + y3 = _mm256_add_epi32(y9, y4); + /* rnd_1: 5 - 5 */ + rdx = (word32)((word32)rdx ^ (word32)rcx); + r12 = (word32)((word32)r12 + (word32)r8); + rbx = (word32)((word32)r10); + y4 = _mm256_add_epi32(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480))); + /* rnd_1: 6 - 6 */ + rbx = (word32)((word32)rbx ^ (word32)r9); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax & (word32)rbx); + /* rnd_1: 7 - 7 */ + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + r8 = (word32)((word32)r8 + (word32)rax); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 480), y4); + rax = (word32)(0); + rcx = (word32)(0); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 384)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 388)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 392)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 396)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 416)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 420)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 424)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 428)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 448)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 452)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 456)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 460)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 480)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 484)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 488)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 492)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + r8 = (word32)((word32)r8 + (word32)rax); + rcx = (word32)(0); + r8 = (word32)((word32)r8 + WC_L32(rdi, 0)); + r9 = (word32)((word32)r9 + WC_L32(rdi, 4)); + r10 = (word32)((word32)r10 + WC_L32(rdi, 8)); + r11 = (word32)((word32)r11 + WC_L32(rdi, 12)); + r12 = (word32)((word32)r12 + WC_L32(rdi, 16)); + r13 = (word32)((word32)r13 + WC_L32(rdi, 20)); + r14 = (word32)((word32)r14 + WC_L32(rdi, 24)); + r15 = (word32)((word32)r15 + WC_L32(rdi, 28)); + WC_S32(rdi, 0) = (word32)((word32)r8); + WC_S32(rdi, 4) = (word32)((word32)r9); + WC_S32(rdi, 8) = (word32)((word32)r10); + WC_S32(rdi, 12) = (word32)((word32)r11); + WC_S32(rdi, 16) = (word32)((word32)r12); + WC_S32(rdi, 20) = (word32)((word32)r13); + WC_S32(rdi, 24) = (word32)((word32)r14); + WC_S32(rdi, 28) = (word32)((word32)r15); + rbx = (word32)((word32)r9); + rax = (word32)(0); + rbx = (word32)((word32)rbx ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 16)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 20)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 24)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 28)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 48)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 52)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 56)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 60)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 80)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 84)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 88)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 92)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 112)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 116)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 120)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 124)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 144)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 148)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 152)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 156)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 176)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 180)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 184)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 188)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 208)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 212)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 216)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 220)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 240)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 244)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 248)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 252)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 272)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 276)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 280)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 284)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 304)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 308)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 312)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 316)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 336)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 340)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 344)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 348)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 368)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 372)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 376)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 380)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 400)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 404)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 408)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 412)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 432)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 436)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 440)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 444)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r12) >> 6) | (((word32)r12) << 26)); + rcx = (word32)((((word32)r12) >> 11) | (((word32)r12) << 21)); + r8 = (word32)(r8 + rax); + r15 = (word32)((word32)r15 + WC_L32(rsp, 464)); + rax = (word32)((word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r12) >> 25) | (((word32)r12) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r12); + r15 = (word32)((word32)r15 + (word32)rdx); + rdx = (word32)((((word32)r8) >> 2) | (((word32)r8) << 30)); + rcx = (word32)((((word32)r8) >> 13) | (((word32)r8) << 19)); + rax = (word32)((word32)rax ^ (word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r8) >> 22) | (((word32)r8) << 10)); + r15 = (word32)((word32)r15 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r9); + r11 = (word32)((word32)r11 + (word32)r15); + rax = (word32)((word32)rax ^ (word32)r8); + rbx = (word32)((word32)rbx & (word32)rax); + r15 = (word32)((word32)r15 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r11) >> 6) | (((word32)r11) << 26)); + rcx = (word32)((((word32)r11) >> 11) | (((word32)r11) << 21)); + r15 = (word32)((word32)r15 + (word32)rbx); + r14 = (word32)((word32)r14 + WC_L32(rsp, 468)); + rbx = (word32)((word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r11) >> 25) | (((word32)r11) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r11); + r14 = (word32)((word32)r14 + (word32)rdx); + rdx = (word32)((((word32)r15) >> 2) | (((word32)r15) << 30)); + rcx = (word32)((((word32)r15) >> 13) | (((word32)r15) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r13); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r15) >> 22) | (((word32)r15) << 10)); + r14 = (word32)((word32)r14 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r8); + r10 = (word32)(r10 + r14); + rbx = (word32)((word32)rbx ^ (word32)r15); + rax = (word32)((word32)rax & (word32)rbx); + r14 = (word32)((word32)r14 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r10) >> 6) | (((word32)r10) << 26)); + rcx = (word32)((((word32)r10) >> 11) | (((word32)r10) << 21)); + r14 = (word32)(r14 + rax); + r13 = (word32)((word32)r13 + WC_L32(rsp, 472)); + rax = (word32)((word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r10) >> 25) | (((word32)r10) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r10); + r13 = (word32)((word32)r13 + (word32)rdx); + rdx = (word32)((((word32)r14) >> 2) | (((word32)r14) << 30)); + rcx = (word32)((((word32)r14) >> 13) | (((word32)r14) << 19)); + rax = (word32)((word32)rax ^ (word32)r12); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r14) >> 22) | (((word32)r14) << 10)); + r13 = (word32)((word32)r13 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r15); + r9 = (word32)((word32)r9 + (word32)r13); + rax = (word32)((word32)rax ^ (word32)r14); + rbx = (word32)((word32)rbx & (word32)rax); + r13 = (word32)((word32)r13 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r9) >> 6) | (((word32)r9) << 26)); + rcx = (word32)((((word32)r9) >> 11) | (((word32)r9) << 21)); + r13 = (word32)((word32)r13 + (word32)rbx); + r12 = (word32)((word32)r12 + WC_L32(rsp, 476)); + rbx = (word32)((word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r9) >> 25) | (((word32)r9) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r9); + r12 = (word32)((word32)r12 + (word32)rdx); + rdx = (word32)((((word32)r13) >> 2) | (((word32)r13) << 30)); + rcx = (word32)((((word32)r13) >> 13) | (((word32)r13) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r11); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r13) >> 22) | (((word32)r13) << 10)); + r12 = (word32)((word32)r12 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r14); + r8 = (word32)(r8 + r12); + rbx = (word32)((word32)rbx ^ (word32)r13); + rax = (word32)((word32)rax & (word32)rbx); + r12 = (word32)((word32)r12 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r14); + rdx = (word32)((((word32)r8) >> 6) | (((word32)r8) << 26)); + rcx = (word32)((((word32)r8) >> 11) | (((word32)r8) << 21)); + r12 = (word32)(r12 + rax); + r11 = (word32)((word32)r11 + WC_L32(rsp, 496)); + rax = (word32)((word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + rdx = (word32)((((word32)r8) >> 25) | (((word32)r8) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r8); + r11 = (word32)((word32)r11 + (word32)rdx); + rdx = (word32)((((word32)r12) >> 2) | (((word32)r12) << 30)); + rcx = (word32)((((word32)r12) >> 13) | (((word32)r12) << 19)); + rax = (word32)((word32)rax ^ (word32)r10); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r12) >> 22) | (((word32)r12) << 10)); + r11 = (word32)((word32)r11 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r13); + r15 = (word32)((word32)r15 + (word32)r11); + rax = (word32)((word32)rax ^ (word32)r12); + rbx = (word32)((word32)rbx & (word32)rax); + r11 = (word32)((word32)r11 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r13); + rdx = (word32)((((word32)r15) >> 6) | (((word32)r15) << 26)); + rcx = (word32)((((word32)r15) >> 11) | (((word32)r15) << 21)); + r11 = (word32)((word32)r11 + (word32)rbx); + r10 = (word32)((word32)r10 + WC_L32(rsp, 500)); + rbx = (word32)((word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r9); + rdx = (word32)((((word32)r15) >> 25) | (((word32)r15) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r15); + r10 = (word32)((word32)r10 + (word32)rdx); + rdx = (word32)((((word32)r11) >> 2) | (((word32)r11) << 30)); + rcx = (word32)((((word32)r11) >> 13) | (((word32)r11) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r9); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r11) >> 22) | (((word32)r11) << 10)); + r10 = (word32)((word32)r10 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r12); + r14 = (word32)(r14 + r10); + rbx = (word32)((word32)rbx ^ (word32)r11); + rax = (word32)((word32)rax & (word32)rbx); + r10 = (word32)((word32)r10 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r12); + rdx = (word32)((((word32)r14) >> 6) | (((word32)r14) << 26)); + rcx = (word32)((((word32)r14) >> 11) | (((word32)r14) << 21)); + r10 = (word32)(r10 + rax); + r9 = (word32)((word32)r9 + WC_L32(rsp, 504)); + rax = (word32)((word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r8); + rdx = (word32)((((word32)r14) >> 25) | (((word32)r14) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)rax & (word32)r14); + r9 = (word32)((word32)r9 + (word32)rdx); + rdx = (word32)((((word32)r10) >> 2) | (((word32)r10) << 30)); + rcx = (word32)((((word32)r10) >> 13) | (((word32)r10) << 19)); + rax = (word32)((word32)rax ^ (word32)r8); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r10) >> 22) | (((word32)r10) << 10)); + r9 = (word32)((word32)r9 + (word32)rax); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rax = (word32)((word32)r11); + r13 = (word32)((word32)r13 + (word32)r9); + rax = (word32)((word32)rax ^ (word32)r10); + rbx = (word32)((word32)rbx & (word32)rax); + r9 = (word32)((word32)r9 + (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r11); + rdx = (word32)((((word32)r13) >> 6) | (((word32)r13) << 26)); + rcx = (word32)((((word32)r13) >> 11) | (((word32)r13) << 21)); + r9 = (word32)((word32)r9 + (word32)rbx); + r8 = (word32)((word32)r8 + WC_L32(rsp, 508)); + rbx = (word32)((word32)r14); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rbx = (word32)((word32)rbx ^ (word32)r15); + rdx = (word32)((((word32)r13) >> 25) | (((word32)r13) << 7)); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)rbx & (word32)r13); + r8 = (word32)((word32)r8 + (word32)rdx); + rdx = (word32)((((word32)r9) >> 2) | (((word32)r9) << 30)); + rcx = (word32)((((word32)r9) >> 13) | (((word32)r9) << 19)); + rbx = (word32)((word32)rbx ^ (word32)r15); + rcx = (word32)((word32)rcx ^ (word32)rdx); + rdx = (word32)((((word32)r9) >> 22) | (((word32)r9) << 10)); + r8 = (word32)((word32)r8 + (word32)rbx); + rdx = (word32)((word32)rdx ^ (word32)rcx); + rbx = (word32)((word32)r10); + r12 = (word32)(r12 + r8); + rbx = (word32)((word32)rbx ^ (word32)r9); + rax = (word32)((word32)rax & (word32)rbx); + r8 = (word32)((word32)r8 + (word32)rdx); + rax = (word32)((word32)rax ^ (word32)r10); + r8 = (word32)((word32)r8 + (word32)rax); + rsi = (word64)(rsi + 0x80); + r8 = (word32)((word32)r8 + WC_L32(rdi, 0)); + r9 = (word32)((word32)r9 + WC_L32(rdi, 4)); + r10 = (word32)((word32)r10 + WC_L32(rdi, 8)); + r11 = (word32)((word32)r11 + WC_L32(rdi, 12)); + r12 = (word32)((word32)r12 + WC_L32(rdi, 16)); + r13 = (word32)((word32)r13 + WC_L32(rdi, 20)); + r14 = (word32)((word32)r14 + WC_L32(rdi, 24)); + r15 = (word32)((word32)r15 + WC_L32(rdi, 28)); + WC_S32(rsp, 512) = (word32)(WC_L32(rsp, 512) - 0x80); + zf2 = (word32)WC_S32(rsp, 512); + WC_S32(rdi, 0) = (word32)((word32)r8); + WC_S32(rdi, 4) = (word32)((word32)r9); + WC_S32(rdi, 8) = (word32)((word32)r10); + WC_S32(rdi, 12) = (word32)((word32)r11); + WC_S32(rdi, 16) = (word32)((word32)r12); + WC_S32(rdi, 20) = (word32)((word32)r13); + WC_S32(rdi, 24) = (word32)((word32)r14); + WC_S32(rdi, 28) = (word32)((word32)r15); + if ((zf2) != (0)) { + goto L_sha256_len_avx2_rorx_start; + } +L_sha256_len_avx2_rorx_done: + rax = (word64)(0); + return (int)(word32)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* WOLFSSL_X86_64_BUILD */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/sha3_x86_64_intrin.c b/wolfcrypt/src/sha3_x86_64_intrin.c new file mode 100644 index 00000000000..ca4807bc703 --- /dev/null +++ b/wolfcrypt/src/sha3_x86_64_intrin.c @@ -0,0 +1,56837 @@ +/* sha3_x86_64_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_SHA3_X86_64_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sha3_block_bmi2(word64* s); +extern WOLFSSL_LOCAL void sha3_block_n_bmi2(word64* s, const byte* data, + word32 n, word64 c); +extern WOLFSSL_LOCAL void sha3_block_avx2(word64* s); +extern WOLFSSL_LOCAL void sha3_block_n_avx2(word64* s, const byte* data, + word32 n, word64 c); +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA) || defined(WOLFSSL_HAVE_FRODOKEM) +extern WOLFSSL_LOCAL void sha3_blocksx4_avx2(word64* s); +#endif +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA) +extern WOLFSSL_LOCAL void sha3_128_blocksx4_seed_avx2(word64* s, byte* seed); +#endif +#ifdef WOLFSSL_HAVE_FRODOKEM +extern WOLFSSL_LOCAL void sha3_blocksx4_out_avx2(word64* s, byte* out, + word32 len); +#endif +#ifdef WOLFSSL_HAVE_MLKEM +extern WOLFSSL_LOCAL void sha3_256_blocksx4_seed_avx2(word64* s, byte* seed); +#endif +#ifdef WOLFSSL_HAVE_MLDSA +extern WOLFSSL_LOCAL void sha3_256_blocksx4_seed_64_avx2(word64* s, byte* seed); +#endif +#ifdef HAVE_INTEL_AVX512 +#ifdef WOLFSSL_HAVE_FRODOKEM +extern WOLFSSL_LOCAL void sha3_blocksx8_out_avx512(word64* s, byte* out, + word32 len); +#endif +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) +extern WOLFSSL_LOCAL void sha3_blocksx8_avx512(word64* s); +extern WOLFSSL_LOCAL void sha3_128_blocksx8_seed_avx512(word64* s, byte* seed); +#endif +#ifdef WOLFSSL_HAVE_MLKEM +extern WOLFSSL_LOCAL void sha3_256_blocksx8_seed_avx512(word64* s, byte* seed); +#endif +#ifdef WOLFSSL_HAVE_MLDSA +extern WOLFSSL_LOCAL void sha3_256_blocksx8_seed_64_avx512(word64* s, + byte* seed); + +#endif +#endif +#endif +XALIGNED(32) static const word64 L_sha3_avx2_r[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000001ULL, + 0x0000000000000001ULL, 0x0000000000000001ULL, + 0x0000000000008082ULL, 0x0000000000008082ULL, + 0x0000000000008082ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x800000000000808aULL, + 0x800000000000808aULL, 0x800000000000808aULL, + 0x8000000080008000ULL, 0x8000000080008000ULL, + 0x8000000080008000ULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x000000000000808bULL, + 0x000000000000808bULL, 0x000000000000808bULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000000008009ULL, 0x8000000000008009ULL, + 0x8000000000008009ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x000000000000008aULL, + 0x000000000000008aULL, 0x000000000000008aULL, + 0x0000000000000088ULL, 0x0000000000000088ULL, + 0x0000000000000088ULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x0000000080008009ULL, + 0x0000000080008009ULL, 0x0000000080008009ULL, + 0x000000008000000aULL, 0x000000008000000aULL, + 0x000000008000000aULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x000000008000808bULL, + 0x000000008000808bULL, 0x000000008000808bULL, + 0x800000000000008bULL, 0x800000000000008bULL, + 0x800000000000008bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008089ULL, + 0x8000000000008089ULL, 0x8000000000008089ULL, + 0x8000000000008003ULL, 0x8000000000008003ULL, + 0x8000000000008003ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000008002ULL, + 0x8000000000008002ULL, 0x8000000000008002ULL, + 0x8000000000000080ULL, 0x8000000000000080ULL, + 0x8000000000000080ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x000000000000800aULL, + 0x000000000000800aULL, 0x000000000000800aULL, + 0x800000008000000aULL, 0x800000008000000aULL, + 0x800000008000000aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000000008080ULL, 0x8000000000008080ULL, + 0x8000000000008080ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x8000000080008008ULL, 0x8000000080008008ULL, + 0x8000000080008008ULL, 0x8000000080008008ULL, +}; + +XALIGNED(32) static const word64 L_sha3_x4_avx2_r[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000001ULL, + 0x0000000000000001ULL, 0x0000000000000001ULL, + 0x0000000000008082ULL, 0x0000000000008082ULL, + 0x0000000000008082ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x800000000000808aULL, + 0x800000000000808aULL, 0x800000000000808aULL, + 0x8000000080008000ULL, 0x8000000080008000ULL, + 0x8000000080008000ULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x000000000000808bULL, + 0x000000000000808bULL, 0x000000000000808bULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000000008009ULL, 0x8000000000008009ULL, + 0x8000000000008009ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x000000000000008aULL, + 0x000000000000008aULL, 0x000000000000008aULL, + 0x0000000000000088ULL, 0x0000000000000088ULL, + 0x0000000000000088ULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x0000000080008009ULL, + 0x0000000080008009ULL, 0x0000000080008009ULL, + 0x000000008000000aULL, 0x000000008000000aULL, + 0x000000008000000aULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x000000008000808bULL, + 0x000000008000808bULL, 0x000000008000808bULL, + 0x800000000000008bULL, 0x800000000000008bULL, + 0x800000000000008bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008089ULL, + 0x8000000000008089ULL, 0x8000000000008089ULL, + 0x8000000000008003ULL, 0x8000000000008003ULL, + 0x8000000000008003ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000008002ULL, + 0x8000000000008002ULL, 0x8000000000008002ULL, + 0x8000000000000080ULL, 0x8000000000000080ULL, + 0x8000000000000080ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x000000000000800aULL, + 0x000000000000800aULL, 0x000000000000800aULL, + 0x800000008000000aULL, 0x800000008000000aULL, + 0x800000008000000aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000000008080ULL, 0x8000000000008080ULL, + 0x8000000000008080ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x8000000080008008ULL, 0x8000000080008008ULL, + 0x8000000080008008ULL, 0x8000000080008008ULL, +}; + +#ifdef HAVE_INTEL_AVX2 +WC_X64I_TARGET("bmi,bmi2") +WOLFSSL_LOCAL void sha3_block_bmi2(word64* s) +{ + word64 rdi, rsi, r10, r11, r12, r13, r14, rdx, rax, rcx, r8, r9, r15; + + rdi = (word64)(size_t)s; + + rsi = (word64)(WC_L64(rdi, 0)); + rdi = (word64)(rdi + 0x60); + /* Round 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -88)); + r12 = (word64)(WC_L64(rdi, -80)); + r13 = (word64)(WC_L64(rdi, -72)); + r14 = (word64)(WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ WC_L64(rdi, -56)); + r11 = (word64)(r11 ^ WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -24)); + r10 = (word64)(r10 ^ WC_L64(rdi, -16)); + r11 = (word64)(r11 ^ WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 0)); + r13 = (word64)(r13 ^ WC_L64(rdi, 8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 16)); + r10 = (word64)(r10 ^ WC_L64(rdi, 24)); + r11 = (word64)(r11 ^ WC_L64(rdi, 32)); + r12 = (word64)(r12 ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 48)); + r14 = (word64)(r14 ^ WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ WC_L64(rdi, 64)); + r11 = (word64)(r11 ^ WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 88)); + r14 = (word64)(r14 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -48)); + r12 = (word64)(WC_L64(rdi, 0)); + r13 = (word64)(WC_L64(rdi, 48)); + r14 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 48) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 96) = (word64)(r13); + /* XOR in constant */ + rsi = (word64)(rsi ^ 1); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -72)); + r11 = (word64)(WC_L64(rdi, -24)); + r12 = (word64)(WC_L64(rdi, -16)); + r13 = (word64)(WC_L64(rdi, 32)); + r14 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 80) = (word64)(r15); + WC_S64(rdi, -72) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -88)); + r11 = (word64)(WC_L64(rdi, -40)); + r12 = (word64)(WC_L64(rdi, 8)); + r13 = (word64)(WC_L64(rdi, 56)); + r14 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 64) = (word64)(r15); + WC_S64(rdi, -88) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -64)); + r11 = (word64)(WC_L64(rdi, -56)); + r12 = (word64)(WC_L64(rdi, -8)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 88) = (word64)(r15); + WC_S64(rdi, -64) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -80)); + r8 = (word64)(r8 ^ WC_L64(rdi, -32)); + r9 = (word64)(r9 ^ WC_L64(rdi, 16)); + rdx = (word64)(rdx ^ WC_L64(rdi, 24)); + rax = (word64)(rax ^ WC_L64(rdi, 72)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -80) = (word64)(r10); + WC_S64(rdi, -32) = (word64)(r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rdi, 72) = (word64)(r14); + /* Round 1 */ + r10 = (word64)(r10 ^ rsi); + r10 = (word64)(r10 ^ WC_L64(rdi, -88)); + r10 = (word64)(r10 ^ WC_L64(rdi, -72)); + r10 = (word64)(r10 ^ WC_L64(rdi, -64)); + r11 = (word64)(r11 ^ WC_L64(rdi, -56)); + r11 = (word64)(r11 ^ WC_L64(rdi, -48)); + r11 = (word64)(r11 ^ WC_L64(rdi, -40)); + r11 = (word64)(r11 ^ WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ WC_L64(rdi, 8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 32)); + r13 = (word64)(r13 ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 48)); + r13 = (word64)(r13 ^ WC_L64(rdi, 56)); + r14 = (word64)(r14 ^ WC_L64(rdi, 64)); + r14 = (word64)(r14 ^ WC_L64(rdi, 80)); + r14 = (word64)(r14 ^ WC_L64(rdi, 88)); + r14 = (word64)(r14 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -24)); + r12 = (word64)(WC_L64(rdi, 8)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 40) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 72) = (word64)(r13); + /* XOR in constant */ + rsi = (word64)(rsi ^ 0x8082); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 48)); + r11 = (word64)(WC_L64(rdi, 80)); + r12 = (word64)(WC_L64(rdi, -88)); + r13 = (word64)(WC_L64(rdi, -56)); + r14 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 16) = (word64)(r15); + WC_S64(rdi, 48) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -48)); + r11 = (word64)(WC_L64(rdi, -16)); + r12 = (word64)(WC_L64(rdi, 56)); + r13 = (word64)(WC_L64(rdi, 88)); + r14 = (word64)(WC_L64(rdi, -80)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -80) = (word64)(r15); + WC_S64(rdi, -48) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 96)); + r11 = (word64)(WC_L64(rdi, -72)); + r12 = (word64)(WC_L64(rdi, -40)); + r13 = (word64)(WC_L64(rdi, -8)); + r14 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 24) = (word64)(r15); + WC_S64(rdi, 96) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 0)); + r8 = (word64)(r8 ^ WC_L64(rdi, 32)); + r9 = (word64)(r9 ^ WC_L64(rdi, 64)); + rdx = (word64)(rdx ^ WC_L64(rdi, -64)); + rax = (word64)(rax ^ WC_L64(rdi, -32)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 0) = (word64)(r10); + WC_S64(rdi, 32) = (word64)(r11); + WC_S64(rdi, 64) = (word64)(r12); + WC_S64(rdi, -64) = (word64)(r13); + WC_S64(rdi, -32) = (word64)(r14); + /* Round 2 */ + r10 = (word64)(r10 ^ rsi); + r12 = (word64)(r12 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r11 = (word64)(r11 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -56)); + r10 = (word64)(r10 ^ WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ WC_L64(rdi, -40)); + r11 = (word64)(r11 ^ WC_L64(rdi, -24)); + r11 = (word64)(r11 ^ WC_L64(rdi, -16)); + r13 = (word64)(r13 ^ WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 16)); + r14 = (word64)(r14 ^ WC_L64(rdi, 24)); + r13 = (word64)(r13 ^ WC_L64(rdi, 40)); + r10 = (word64)(r10 ^ WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ WC_L64(rdi, 56)); + r14 = (word64)(r14 ^ WC_L64(rdi, 72)); + r11 = (word64)(r11 ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 88)); + r10 = (word64)(r10 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 80)); + r12 = (word64)(WC_L64(rdi, 56)); + r13 = (word64)(WC_L64(rdi, -8)); + r14 = (word64)(WC_L64(rdi, -32)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -8) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -32) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x800000000000808a); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 40)); + r11 = (word64)(WC_L64(rdi, 16)); + r12 = (word64)(WC_L64(rdi, -48)); + r13 = (word64)(WC_L64(rdi, -72)); + r14 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 64) = (word64)(r15); + WC_S64(rdi, 40) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -24)); + r11 = (word64)(WC_L64(rdi, -88)); + r12 = (word64)(WC_L64(rdi, 88)); + r13 = (word64)(WC_L64(rdi, 24)); + r14 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 0) = (word64)(r15); + WC_S64(rdi, -24) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 72)); + r11 = (word64)(WC_L64(rdi, 48)); + r12 = (word64)(WC_L64(rdi, -16)); + r13 = (word64)(WC_L64(rdi, -40)); + r14 = (word64)(WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -64) = (word64)(r15); + WC_S64(rdi, 72) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 8)); + r8 = (word64)(r8 ^ WC_L64(rdi, -56)); + r9 = (word64)(r9 ^ WC_L64(rdi, -80)); + rdx = (word64)(rdx ^ WC_L64(rdi, 96)); + rax = (word64)(rax ^ WC_L64(rdi, 32)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, -56) = (word64)(r11); + WC_S64(rdi, -80) = (word64)(r12); + WC_S64(rdi, 96) = (word64)(r13); + WC_S64(rdi, 32) = (word64)(r14); + /* Round 3 */ + r10 = (word64)(r10 ^ rsi); + r11 = (word64)(r11 ^ WC_L64(rdi, -88)); + r13 = (word64)(r13 ^ WC_L64(rdi, -72)); + r14 = (word64)(r14 ^ WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ WC_L64(rdi, -48)); + r13 = (word64)(r13 ^ WC_L64(rdi, -40)); + r14 = (word64)(r14 ^ WC_L64(rdi, -32)); + r10 = (word64)(r10 ^ WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ WC_L64(rdi, -16)); + r13 = (word64)(r13 ^ WC_L64(rdi, -8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 0)); + r11 = (word64)(r11 ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 24)); + r10 = (word64)(r10 ^ WC_L64(rdi, 40)); + r11 = (word64)(r11 ^ WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ WC_L64(rdi, 56)); + r14 = (word64)(r14 ^ WC_L64(rdi, 64)); + r10 = (word64)(r10 ^ WC_L64(rdi, 72)); + r11 = (word64)(r11 ^ WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ WC_L64(rdi, 88)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 16)); + r12 = (word64)(WC_L64(rdi, 88)); + r13 = (word64)(WC_L64(rdi, -40)); + r14 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -40) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 32) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000080008000); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -8)); + r11 = (word64)(WC_L64(rdi, 64)); + r12 = (word64)(WC_L64(rdi, -24)); + r13 = (word64)(WC_L64(rdi, 48)); + r14 = (word64)(WC_L64(rdi, -80)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -80) = (word64)(r15); + WC_S64(rdi, -8) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 80)); + r11 = (word64)(WC_L64(rdi, -48)); + r12 = (word64)(WC_L64(rdi, 24)); + r13 = (word64)(WC_L64(rdi, -64)); + r14 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 8) = (word64)(r15); + WC_S64(rdi, 80) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -32)); + r11 = (word64)(WC_L64(rdi, 40)); + r12 = (word64)(WC_L64(rdi, -88)); + r13 = (word64)(WC_L64(rdi, -16)); + r14 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 96) = (word64)(r15); + WC_S64(rdi, -32) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 56)); + r8 = (word64)(r8 ^ WC_L64(rdi, -72)); + r9 = (word64)(r9 ^ WC_L64(rdi, 0)); + rdx = (word64)(rdx ^ WC_L64(rdi, 72)); + rax = (word64)(rax ^ WC_L64(rdi, -56)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 56) = (word64)(r10); + WC_S64(rdi, -72) = (word64)(r11); + WC_S64(rdi, 0) = (word64)(r12); + WC_S64(rdi, 72) = (word64)(r13); + WC_S64(rdi, -56) = (word64)(r14); + /* Round 4 */ + r10 = (word64)(r10 ^ rsi); + r12 = (word64)(r12 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r13 = (word64)(r13 ^ WC_L64(rdi, -64)); + r11 = (word64)(r11 ^ WC_L64(rdi, -48)); + r13 = (word64)(r13 ^ WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ WC_L64(rdi, -24)); + r13 = (word64)(r13 ^ WC_L64(rdi, -16)); + r10 = (word64)(r10 ^ WC_L64(rdi, -8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 8)); + r11 = (word64)(r11 ^ WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 32)); + r11 = (word64)(r11 ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 48)); + r11 = (word64)(r11 ^ WC_L64(rdi, 64)); + r10 = (word64)(r10 ^ WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ WC_L64(rdi, 88)); + r14 = (word64)(r14 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 64)); + r12 = (word64)(WC_L64(rdi, 24)); + r13 = (word64)(WC_L64(rdi, -16)); + r14 = (word64)(WC_L64(rdi, -56)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -16) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -56) = (word64)(r13); + /* XOR in constant */ + rsi = (word64)(rsi ^ 0x808b); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -40)); + r11 = (word64)(WC_L64(rdi, -80)); + r12 = (word64)(WC_L64(rdi, 80)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 0) = (word64)(r15); + WC_S64(rdi, -40) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, -24)); + r12 = (word64)(WC_L64(rdi, -64)); + r13 = (word64)(WC_L64(rdi, 96)); + r14 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 56) = (word64)(r15); + WC_S64(rdi, 16) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 32)); + r11 = (word64)(WC_L64(rdi, -8)); + r12 = (word64)(WC_L64(rdi, -48)); + r13 = (word64)(WC_L64(rdi, -88)); + r14 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 72) = (word64)(r15); + WC_S64(rdi, 32) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 88)); + r8 = (word64)(r8 ^ WC_L64(rdi, 48)); + r9 = (word64)(r9 ^ WC_L64(rdi, 8)); + rdx = (word64)(rdx ^ WC_L64(rdi, -32)); + rax = (word64)(rax ^ WC_L64(rdi, -72)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 88) = (word64)(r10); + WC_S64(rdi, 48) = (word64)(r11); + WC_S64(rdi, 8) = (word64)(r12); + WC_S64(rdi, -32) = (word64)(r13); + WC_S64(rdi, -72) = (word64)(r14); + /* Round 5 */ + r10 = (word64)(r10 ^ rsi); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r11 = (word64)(r11 ^ WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ WC_L64(rdi, -64)); + r14 = (word64)(r14 ^ WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ WC_L64(rdi, -48)); + r10 = (word64)(r10 ^ WC_L64(rdi, -40)); + r11 = (word64)(r11 ^ WC_L64(rdi, -24)); + r13 = (word64)(r13 ^ WC_L64(rdi, -16)); + r11 = (word64)(r11 ^ WC_L64(rdi, -8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 0)); + r10 = (word64)(r10 ^ WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ WC_L64(rdi, 24)); + r10 = (word64)(r10 ^ WC_L64(rdi, 32)); + r13 = (word64)(r13 ^ WC_L64(rdi, 40)); + r14 = (word64)(r14 ^ WC_L64(rdi, 56)); + r11 = (word64)(r11 ^ WC_L64(rdi, 64)); + r14 = (word64)(r14 ^ WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -80)); + r12 = (word64)(WC_L64(rdi, -64)); + r13 = (word64)(WC_L64(rdi, -88)); + r14 = (word64)(WC_L64(rdi, -72)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -88) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -72) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x80000001); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -16)); + r11 = (word64)(WC_L64(rdi, 0)); + r12 = (word64)(WC_L64(rdi, 16)); + r13 = (word64)(WC_L64(rdi, -8)); + r14 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 8) = (word64)(r15); + WC_S64(rdi, -16) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 64)); + r11 = (word64)(WC_L64(rdi, 80)); + r12 = (word64)(WC_L64(rdi, 96)); + r13 = (word64)(WC_L64(rdi, 72)); + r14 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 88) = (word64)(r15); + WC_S64(rdi, 64) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -56)); + r11 = (word64)(WC_L64(rdi, -40)); + r12 = (word64)(WC_L64(rdi, -24)); + r13 = (word64)(WC_L64(rdi, -48)); + r14 = (word64)(WC_L64(rdi, -32)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -32) = (word64)(r15); + WC_S64(rdi, -56) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 24)); + r8 = (word64)(r8 ^ WC_L64(rdi, 40)); + r9 = (word64)(r9 ^ WC_L64(rdi, 56)); + rdx = (word64)(rdx ^ WC_L64(rdi, 32)); + rax = (word64)(rax ^ WC_L64(rdi, 48)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 24) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(r11); + WC_S64(rdi, 56) = (word64)(r12); + WC_S64(rdi, 32) = (word64)(r13); + WC_S64(rdi, 48) = (word64)(r14); + /* Round 6 */ + r10 = (word64)(r10 ^ rsi); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r11 = (word64)(r11 ^ WC_L64(rdi, -80)); + r14 = (word64)(r14 ^ WC_L64(rdi, -72)); + r12 = (word64)(r12 ^ WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ WC_L64(rdi, -56)); + r13 = (word64)(r13 ^ WC_L64(rdi, -48)); + r11 = (word64)(r11 ^ WC_L64(rdi, -40)); + r14 = (word64)(r14 ^ WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ WC_L64(rdi, -24)); + r10 = (word64)(r10 ^ WC_L64(rdi, -16)); + r13 = (word64)(r13 ^ WC_L64(rdi, -8)); + r11 = (word64)(r11 ^ WC_L64(rdi, 0)); + r14 = (word64)(r14 ^ WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 16)); + r10 = (word64)(r10 ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 72)); + r11 = (word64)(r11 ^ WC_L64(rdi, 80)); + r14 = (word64)(r14 ^ WC_L64(rdi, 88)); + r12 = (word64)(r12 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 0)); + r12 = (word64)(WC_L64(rdi, 96)); + r13 = (word64)(WC_L64(rdi, -48)); + r14 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -48) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 48) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000080008081); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -88)); + r11 = (word64)(WC_L64(rdi, 8)); + r12 = (word64)(WC_L64(rdi, 64)); + r13 = (word64)(WC_L64(rdi, -40)); + r14 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 56) = (word64)(r15); + WC_S64(rdi, -88) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -80)); + r11 = (word64)(WC_L64(rdi, 16)); + r12 = (word64)(WC_L64(rdi, 72)); + r13 = (word64)(WC_L64(rdi, -32)); + r14 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 24) = (word64)(r15); + WC_S64(rdi, -80) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -72)); + r11 = (word64)(WC_L64(rdi, -16)); + r12 = (word64)(WC_L64(rdi, 80)); + r13 = (word64)(WC_L64(rdi, -24)); + r14 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 32) = (word64)(r15); + WC_S64(rdi, -72) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -64)); + r8 = (word64)(r8 ^ WC_L64(rdi, -8)); + r9 = (word64)(r9 ^ WC_L64(rdi, 88)); + rdx = (word64)(rdx ^ WC_L64(rdi, -56)); + rax = (word64)(rax ^ WC_L64(rdi, 40)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -64) = (word64)(r10); + WC_S64(rdi, -8) = (word64)(r11); + WC_S64(rdi, 88) = (word64)(r12); + WC_S64(rdi, -56) = (word64)(r13); + WC_S64(rdi, 40) = (word64)(r14); + /* Round 7 */ + r10 = (word64)(r10 ^ rsi); + r10 = (word64)(r10 ^ WC_L64(rdi, -88)); + r10 = (word64)(r10 ^ WC_L64(rdi, -80)); + r10 = (word64)(r10 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -48)); + r13 = (word64)(r13 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -32)); + r13 = (word64)(r13 ^ WC_L64(rdi, -24)); + r11 = (word64)(r11 ^ WC_L64(rdi, -16)); + r11 = (word64)(r11 ^ WC_L64(rdi, 0)); + r11 = (word64)(r11 ^ WC_L64(rdi, 8)); + r11 = (word64)(r11 ^ WC_L64(rdi, 16)); + r14 = (word64)(r14 ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 48)); + r14 = (word64)(r14 ^ WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 8)); + r12 = (word64)(WC_L64(rdi, 72)); + r13 = (word64)(WC_L64(rdi, -24)); + r14 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -24) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 40) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000000008009); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -48)); + r11 = (word64)(WC_L64(rdi, 56)); + r12 = (word64)(WC_L64(rdi, -80)); + r13 = (word64)(WC_L64(rdi, -16)); + r14 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 88) = (word64)(r15); + WC_S64(rdi, -48) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 0)); + r11 = (word64)(WC_L64(rdi, 64)); + r12 = (word64)(WC_L64(rdi, -32)); + r13 = (word64)(WC_L64(rdi, 32)); + r14 = (word64)(WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -64) = (word64)(r15); + WC_S64(rdi, 0) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 48)); + r11 = (word64)(WC_L64(rdi, -88)); + r12 = (word64)(WC_L64(rdi, 16)); + r13 = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(WC_L64(rdi, -56)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -56) = (word64)(r15); + WC_S64(rdi, 48) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 96)); + r8 = (word64)(r8 ^ WC_L64(rdi, -40)); + r9 = (word64)(r9 ^ WC_L64(rdi, 24)); + rdx = (word64)(rdx ^ WC_L64(rdi, -72)); + rax = (word64)(rax ^ WC_L64(rdi, -8)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 96) = (word64)(r10); + WC_S64(rdi, -40) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + WC_S64(rdi, -72) = (word64)(r13); + WC_S64(rdi, -8) = (word64)(r14); + /* Round 8 */ + r10 = (word64)(r10 ^ rsi); + r11 = (word64)(r11 ^ WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ WC_L64(rdi, -80)); + r14 = (word64)(r14 ^ WC_L64(rdi, -64)); + r14 = (word64)(r14 ^ WC_L64(rdi, -56)); + r10 = (word64)(r10 ^ WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ WC_L64(rdi, -32)); + r13 = (word64)(r13 ^ WC_L64(rdi, -24)); + r13 = (word64)(r13 ^ WC_L64(rdi, -16)); + r10 = (word64)(r10 ^ WC_L64(rdi, 0)); + r11 = (word64)(r11 ^ WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 40)); + r10 = (word64)(r10 ^ WC_L64(rdi, 48)); + r11 = (word64)(r11 ^ WC_L64(rdi, 56)); + r11 = (word64)(r11 ^ WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ WC_L64(rdi, 72)); + r13 = (word64)(r13 ^ WC_L64(rdi, 80)); + r14 = (word64)(r14 ^ WC_L64(rdi, 88)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 56)); + r12 = (word64)(WC_L64(rdi, -32)); + r13 = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(WC_L64(rdi, -8)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 80) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -8) = (word64)(r13); + /* XOR in constant */ + rsi = (word64)(rsi ^ 0x8a); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -24)); + r11 = (word64)(WC_L64(rdi, 88)); + r12 = (word64)(WC_L64(rdi, 0)); + r13 = (word64)(WC_L64(rdi, -88)); + r14 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 24) = (word64)(r15); + WC_S64(rdi, -24) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 8)); + r11 = (word64)(WC_L64(rdi, -80)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, -56)); + r14 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 96) = (word64)(r15); + WC_S64(rdi, 8) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 40)); + r11 = (word64)(WC_L64(rdi, -48)); + r12 = (word64)(WC_L64(rdi, 64)); + r13 = (word64)(WC_L64(rdi, 16)); + r14 = (word64)(WC_L64(rdi, -72)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -72) = (word64)(r15); + WC_S64(rdi, 40) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 72)); + r8 = (word64)(r8 ^ WC_L64(rdi, -16)); + r9 = (word64)(r9 ^ WC_L64(rdi, -64)); + rdx = (word64)(rdx ^ WC_L64(rdi, 48)); + rax = (word64)(rax ^ WC_L64(rdi, -40)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 72) = (word64)(r10); + WC_S64(rdi, -16) = (word64)(r11); + WC_S64(rdi, -64) = (word64)(r12); + WC_S64(rdi, 48) = (word64)(r13); + WC_S64(rdi, -40) = (word64)(r14); + /* Round 9 */ + r10 = (word64)(r10 ^ rsi); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r11 = (word64)(r11 ^ WC_L64(rdi, -80)); + r14 = (word64)(r14 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -56)); + r11 = (word64)(r11 ^ WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ WC_L64(rdi, -32)); + r10 = (word64)(r10 ^ WC_L64(rdi, -24)); + r14 = (word64)(r14 ^ WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 0)); + r10 = (word64)(r10 ^ WC_L64(rdi, 8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 16)); + r14 = (word64)(r14 ^ WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 32)); + r10 = (word64)(r10 ^ WC_L64(rdi, 40)); + r11 = (word64)(r11 ^ WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 80)); + r11 = (word64)(r11 ^ WC_L64(rdi, 88)); + r14 = (word64)(r14 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 88)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 16)); + r14 = (word64)(WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 16) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -40) = (word64)(r13); + /* XOR in constant */ + rsi = (word64)(rsi ^ 0x88); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 80)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 8)); + r13 = (word64)(WC_L64(rdi, -48)); + r14 = (word64)(WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -64) = (word64)(r15); + WC_S64(rdi, 80) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 56)); + r11 = (word64)(WC_L64(rdi, 0)); + r12 = (word64)(WC_L64(rdi, -56)); + r13 = (word64)(WC_L64(rdi, -72)); + r14 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 72) = (word64)(r15); + WC_S64(rdi, 56) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -8)); + r11 = (word64)(WC_L64(rdi, -24)); + r12 = (word64)(WC_L64(rdi, -80)); + r13 = (word64)(WC_L64(rdi, 64)); + r14 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 48) = (word64)(r15); + WC_S64(rdi, -8) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -32)); + r8 = (word64)(r8 ^ WC_L64(rdi, -88)); + r9 = (word64)(r9 ^ WC_L64(rdi, 96)); + rdx = (word64)(rdx ^ WC_L64(rdi, 40)); + rax = (word64)(rax ^ WC_L64(rdi, -16)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -32) = (word64)(r10); + WC_S64(rdi, -88) = (word64)(r11); + WC_S64(rdi, 96) = (word64)(r12); + WC_S64(rdi, 40) = (word64)(r13); + WC_S64(rdi, -16) = (word64)(r14); + /* Round 10 */ + r10 = (word64)(r10 ^ rsi); + r12 = (word64)(r12 ^ WC_L64(rdi, -80)); + r13 = (word64)(r13 ^ WC_L64(rdi, -72)); + r14 = (word64)(r14 ^ WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ WC_L64(rdi, -56)); + r13 = (word64)(r13 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -40)); + r11 = (word64)(r11 ^ WC_L64(rdi, -24)); + r10 = (word64)(r10 ^ WC_L64(rdi, -8)); + r11 = (word64)(r11 ^ WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ WC_L64(rdi, 8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 16)); + r11 = (word64)(r11 ^ WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 48)); + r10 = (word64)(r10 ^ WC_L64(rdi, 56)); + r13 = (word64)(r13 ^ WC_L64(rdi, 64)); + r14 = (word64)(r14 ^ WC_L64(rdi, 72)); + r10 = (word64)(r10 ^ WC_L64(rdi, 80)); + r11 = (word64)(r11 ^ WC_L64(rdi, 88)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, -56)); + r13 = (word64)(WC_L64(rdi, 64)); + r14 = (word64)(WC_L64(rdi, -16)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 64) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -16) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x80008009); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, -64)); + r12 = (word64)(WC_L64(rdi, 56)); + r13 = (word64)(WC_L64(rdi, -24)); + r14 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 96) = (word64)(r15); + WC_S64(rdi, 16) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 88)); + r11 = (word64)(WC_L64(rdi, 8)); + r12 = (word64)(WC_L64(rdi, -72)); + r13 = (word64)(WC_L64(rdi, 48)); + r14 = (word64)(WC_L64(rdi, -32)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -32) = (word64)(r15); + WC_S64(rdi, 88) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -40)); + r11 = (word64)(WC_L64(rdi, 80)); + r12 = (word64)(WC_L64(rdi, 0)); + r13 = (word64)(WC_L64(rdi, -80)); + r14 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 40) = (word64)(r15); + WC_S64(rdi, -40) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 32)); + r8 = (word64)(r8 ^ WC_L64(rdi, -48)); + r9 = (word64)(r9 ^ WC_L64(rdi, 72)); + rdx = (word64)(rdx ^ WC_L64(rdi, -8)); + rax = (word64)(rax ^ WC_L64(rdi, -88)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, -48) = (word64)(r11); + WC_S64(rdi, 72) = (word64)(r12); + WC_S64(rdi, -8) = (word64)(r13); + WC_S64(rdi, -88) = (word64)(r14); + /* Round 11 */ + r10 = (word64)(r10 ^ rsi); + r13 = (word64)(r13 ^ WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ WC_L64(rdi, -72)); + r11 = (word64)(r11 ^ WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ WC_L64(rdi, -56)); + r10 = (word64)(r10 ^ WC_L64(rdi, -40)); + r14 = (word64)(r14 ^ WC_L64(rdi, -32)); + r13 = (word64)(r13 ^ WC_L64(rdi, -24)); + r14 = (word64)(r14 ^ WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ WC_L64(rdi, 0)); + r11 = (word64)(r11 ^ WC_L64(rdi, 8)); + r10 = (word64)(r10 ^ WC_L64(rdi, 16)); + r11 = (word64)(r11 ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ WC_L64(rdi, 56)); + r13 = (word64)(r13 ^ WC_L64(rdi, 64)); + r11 = (word64)(r11 ^ WC_L64(rdi, 80)); + r10 = (word64)(r10 ^ WC_L64(rdi, 88)); + r14 = (word64)(r14 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -64)); + r12 = (word64)(WC_L64(rdi, -72)); + r13 = (word64)(WC_L64(rdi, -80)); + r14 = (word64)(WC_L64(rdi, -88)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -80) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -88) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000a); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 64)); + r11 = (word64)(WC_L64(rdi, 96)); + r12 = (word64)(WC_L64(rdi, 88)); + r13 = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 72) = (word64)(r15); + WC_S64(rdi, 64) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 24)); + r11 = (word64)(WC_L64(rdi, 56)); + r12 = (word64)(WC_L64(rdi, 48)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 32) = (word64)(r15); + WC_S64(rdi, 24) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -16)); + r11 = (word64)(WC_L64(rdi, 16)); + r12 = (word64)(WC_L64(rdi, 8)); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, -8)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -8) = (word64)(r15); + WC_S64(rdi, -16) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -56)); + r8 = (word64)(r8 ^ WC_L64(rdi, -24)); + r9 = (word64)(r9 ^ WC_L64(rdi, -32)); + rdx = (word64)(rdx ^ WC_L64(rdi, -40)); + rax = (word64)(rax ^ WC_L64(rdi, -48)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -56) = (word64)(r10); + WC_S64(rdi, -24) = (word64)(r11); + WC_S64(rdi, -32) = (word64)(r12); + WC_S64(rdi, -40) = (word64)(r13); + WC_S64(rdi, -48) = (word64)(r14); + /* Round 12 */ + r10 = (word64)(r10 ^ rsi); + r14 = (word64)(r14 ^ WC_L64(rdi, -88)); + r13 = (word64)(r13 ^ WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ WC_L64(rdi, -72)); + r11 = (word64)(r11 ^ WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ WC_L64(rdi, -16)); + r14 = (word64)(r14 ^ WC_L64(rdi, -8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ WC_L64(rdi, 8)); + r11 = (word64)(r11 ^ WC_L64(rdi, 16)); + r10 = (word64)(r10 ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 32)); + r13 = (word64)(r13 ^ WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ WC_L64(rdi, 48)); + r11 = (word64)(r11 ^ WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ WC_L64(rdi, 64)); + r14 = (word64)(r14 ^ WC_L64(rdi, 72)); + r13 = (word64)(r13 ^ WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ WC_L64(rdi, 88)); + r11 = (word64)(r11 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 96)); + r12 = (word64)(WC_L64(rdi, 48)); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, -48)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 0) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -48) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000808b); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -80)); + r11 = (word64)(WC_L64(rdi, 72)); + r12 = (word64)(WC_L64(rdi, 24)); + r13 = (word64)(WC_L64(rdi, 16)); + r14 = (word64)(WC_L64(rdi, -32)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -32) = (word64)(r15); + WC_S64(rdi, -80) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -64)); + r11 = (word64)(WC_L64(rdi, 88)); + r12 = (word64)(WC_L64(rdi, 40)); + r13 = (word64)(WC_L64(rdi, -8)); + r14 = (word64)(WC_L64(rdi, -56)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -56) = (word64)(r15); + WC_S64(rdi, -64) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -88)); + r11 = (word64)(WC_L64(rdi, 64)); + r12 = (word64)(WC_L64(rdi, 56)); + r13 = (word64)(WC_L64(rdi, 8)); + r14 = (word64)(WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -40) = (word64)(r15); + WC_S64(rdi, -88) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -72)); + r8 = (word64)(r8 ^ WC_L64(rdi, 80)); + r9 = (word64)(r9 ^ WC_L64(rdi, 32)); + rdx = (word64)(rdx ^ WC_L64(rdi, -16)); + rax = (word64)(rax ^ WC_L64(rdi, -24)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -72) = (word64)(r10); + WC_S64(rdi, 80) = (word64)(r11); + WC_S64(rdi, 32) = (word64)(r12); + WC_S64(rdi, -16) = (word64)(r13); + WC_S64(rdi, -24) = (word64)(r14); + /* Round 13 */ + r10 = (word64)(r10 ^ rsi); + r10 = (word64)(r10 ^ WC_L64(rdi, -88)); + r10 = (word64)(r10 ^ WC_L64(rdi, -80)); + r10 = (word64)(r10 ^ WC_L64(rdi, -64)); + r14 = (word64)(r14 ^ WC_L64(rdi, -56)); + r14 = (word64)(r14 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -40)); + r14 = (word64)(r14 ^ WC_L64(rdi, -32)); + r13 = (word64)(r13 ^ WC_L64(rdi, -8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 0)); + r13 = (word64)(r13 ^ WC_L64(rdi, 8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ WC_L64(rdi, 56)); + r11 = (word64)(r11 ^ WC_L64(rdi, 64)); + r11 = (word64)(r11 ^ WC_L64(rdi, 72)); + r11 = (word64)(r11 ^ WC_L64(rdi, 88)); + r11 = (word64)(r11 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 72)); + r12 = (word64)(WC_L64(rdi, 40)); + r13 = (word64)(WC_L64(rdi, 8)); + r14 = (word64)(WC_L64(rdi, -24)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 8) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -24) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x800000000000008b); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 0)); + r11 = (word64)(WC_L64(rdi, -32)); + r12 = (word64)(WC_L64(rdi, -64)); + r13 = (word64)(WC_L64(rdi, 64)); + r14 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 32) = (word64)(r15); + WC_S64(rdi, 0) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 96)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, -8)); + r13 = (word64)(WC_L64(rdi, -40)); + r14 = (word64)(WC_L64(rdi, -72)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -72) = (word64)(r15); + WC_S64(rdi, 96) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -48)); + r11 = (word64)(WC_L64(rdi, -80)); + r12 = (word64)(WC_L64(rdi, 88)); + r13 = (word64)(WC_L64(rdi, 56)); + r14 = (word64)(WC_L64(rdi, -16)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -16) = (word64)(r15); + WC_S64(rdi, -48) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 48)); + r8 = (word64)(r8 ^ WC_L64(rdi, 16)); + r9 = (word64)(r9 ^ WC_L64(rdi, -56)); + rdx = (word64)(rdx ^ WC_L64(rdi, -88)); + rax = (word64)(rax ^ WC_L64(rdi, 80)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 48) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, -56) = (word64)(r12); + WC_S64(rdi, -88) = (word64)(r13); + WC_S64(rdi, 80) = (word64)(r14); + /* Round 14 */ + r10 = (word64)(r10 ^ rsi); + r11 = (word64)(r11 ^ WC_L64(rdi, -80)); + r14 = (word64)(r14 ^ WC_L64(rdi, -72)); + r12 = (word64)(r12 ^ WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ WC_L64(rdi, -48)); + r13 = (word64)(r13 ^ WC_L64(rdi, -40)); + r11 = (word64)(r11 ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -24)); + r14 = (word64)(r14 ^ WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ WC_L64(rdi, -8)); + r10 = (word64)(r10 ^ WC_L64(rdi, 0)); + r13 = (word64)(r13 ^ WC_L64(rdi, 8)); + r11 = (word64)(r11 ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 32)); + r12 = (word64)(r12 ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 56)); + r13 = (word64)(r13 ^ WC_L64(rdi, 64)); + r11 = (word64)(r11 ^ WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ WC_L64(rdi, 88)); + r10 = (word64)(r10 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -32)); + r12 = (word64)(WC_L64(rdi, -8)); + r13 = (word64)(WC_L64(rdi, 56)); + r14 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 56) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 80) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000000008089); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 8)); + r11 = (word64)(WC_L64(rdi, 32)); + r12 = (word64)(WC_L64(rdi, 96)); + r13 = (word64)(WC_L64(rdi, -80)); + r14 = (word64)(WC_L64(rdi, -56)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -56) = (word64)(r15); + WC_S64(rdi, 8) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 72)); + r11 = (word64)(WC_L64(rdi, -64)); + r12 = (word64)(WC_L64(rdi, -40)); + r13 = (word64)(WC_L64(rdi, -16)); + r14 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 48) = (word64)(r15); + WC_S64(rdi, 72) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -24)); + r11 = (word64)(WC_L64(rdi, 0)); + r12 = (word64)(WC_L64(rdi, 24)); + r13 = (word64)(WC_L64(rdi, 88)); + r14 = (word64)(WC_L64(rdi, -88)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -88) = (word64)(r15); + WC_S64(rdi, -24) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 40)); + r8 = (word64)(r8 ^ WC_L64(rdi, 64)); + r9 = (word64)(r9 ^ WC_L64(rdi, -72)); + rdx = (word64)(rdx ^ WC_L64(rdi, -48)); + rax = (word64)(rax ^ WC_L64(rdi, 16)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 40) = (word64)(r10); + WC_S64(rdi, 64) = (word64)(r11); + WC_S64(rdi, -72) = (word64)(r12); + WC_S64(rdi, -48) = (word64)(r13); + WC_S64(rdi, 16) = (word64)(r14); + /* Round 15 */ + r10 = (word64)(r10 ^ rsi); + r14 = (word64)(r14 ^ WC_L64(rdi, -88)); + r13 = (word64)(r13 ^ WC_L64(rdi, -80)); + r11 = (word64)(r11 ^ WC_L64(rdi, -64)); + r14 = (word64)(r14 ^ WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ WC_L64(rdi, -40)); + r11 = (word64)(r11 ^ WC_L64(rdi, -32)); + r10 = (word64)(r10 ^ WC_L64(rdi, -24)); + r13 = (word64)(r13 ^ WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ WC_L64(rdi, -8)); + r11 = (word64)(r11 ^ WC_L64(rdi, 0)); + r10 = (word64)(r10 ^ WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 24)); + r11 = (word64)(r11 ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 48)); + r13 = (word64)(r13 ^ WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 88)); + r12 = (word64)(r12 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 32)); + r12 = (word64)(WC_L64(rdi, -40)); + r13 = (word64)(WC_L64(rdi, 88)); + r14 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 88) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 16) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000000008003); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 56)); + r11 = (word64)(WC_L64(rdi, -56)); + r12 = (word64)(WC_L64(rdi, 72)); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, -72)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -72) = (word64)(r15); + WC_S64(rdi, 56) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -32)); + r11 = (word64)(WC_L64(rdi, 96)); + r12 = (word64)(WC_L64(rdi, -16)); + r13 = (word64)(WC_L64(rdi, -88)); + r14 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 40) = (word64)(r15); + WC_S64(rdi, -32) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 80)); + r11 = (word64)(WC_L64(rdi, 8)); + r12 = (word64)(WC_L64(rdi, -64)); + r13 = (word64)(WC_L64(rdi, 24)); + r14 = (word64)(WC_L64(rdi, -48)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -48) = (word64)(r15); + WC_S64(rdi, 80) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -8)); + r8 = (word64)(r8 ^ WC_L64(rdi, -80)); + r9 = (word64)(r9 ^ WC_L64(rdi, 48)); + rdx = (word64)(rdx ^ WC_L64(rdi, -24)); + rax = (word64)(rax ^ WC_L64(rdi, 64)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -8) = (word64)(r10); + WC_S64(rdi, -80) = (word64)(r11); + WC_S64(rdi, 48) = (word64)(r12); + WC_S64(rdi, -24) = (word64)(r13); + WC_S64(rdi, 64) = (word64)(r14); + /* Round 16 */ + r10 = (word64)(r10 ^ rsi); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -72)); + r12 = (word64)(r12 ^ WC_L64(rdi, -64)); + r11 = (word64)(r11 ^ WC_L64(rdi, -56)); + r14 = (word64)(r14 ^ WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ WC_L64(rdi, -16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 0)); + r11 = (word64)(r11 ^ WC_L64(rdi, 8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 24)); + r11 = (word64)(r11 ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 40)); + r10 = (word64)(r10 ^ WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ WC_L64(rdi, 72)); + r10 = (word64)(r10 ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 88)); + r11 = (word64)(r11 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -56)); + r12 = (word64)(WC_L64(rdi, -16)); + r13 = (word64)(WC_L64(rdi, 24)); + r14 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 24) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 64) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000000008002); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 88)); + r11 = (word64)(WC_L64(rdi, -72)); + r12 = (word64)(WC_L64(rdi, -32)); + r13 = (word64)(WC_L64(rdi, 8)); + r14 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 48) = (word64)(r15); + WC_S64(rdi, 88) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 32)); + r11 = (word64)(WC_L64(rdi, 72)); + r12 = (word64)(WC_L64(rdi, -88)); + r13 = (word64)(WC_L64(rdi, -48)); + r14 = (word64)(WC_L64(rdi, -8)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -8) = (word64)(r15); + WC_S64(rdi, 32) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 56)); + r12 = (word64)(WC_L64(rdi, 96)); + r13 = (word64)(WC_L64(rdi, -64)); + r14 = (word64)(WC_L64(rdi, -24)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -24) = (word64)(r15); + WC_S64(rdi, 16) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -40)); + r8 = (word64)(r8 ^ WC_L64(rdi, 0)); + r9 = (word64)(r9 ^ WC_L64(rdi, 40)); + rdx = (word64)(rdx ^ WC_L64(rdi, 80)); + rax = (word64)(rax ^ WC_L64(rdi, -80)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -40) = (word64)(r10); + WC_S64(rdi, 0) = (word64)(r11); + WC_S64(rdi, 40) = (word64)(r12); + WC_S64(rdi, 80) = (word64)(r13); + WC_S64(rdi, -80) = (word64)(r14); + /* Round 17 */ + r10 = (word64)(r10 ^ rsi); + r12 = (word64)(r12 ^ WC_L64(rdi, -88)); + r11 = (word64)(r11 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -64)); + r11 = (word64)(r11 ^ WC_L64(rdi, -56)); + r13 = (word64)(r13 ^ WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ WC_L64(rdi, -16)); + r14 = (word64)(r14 ^ WC_L64(rdi, -8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 8)); + r10 = (word64)(r10 ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 24)); + r10 = (word64)(r10 ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 48)); + r11 = (word64)(r11 ^ WC_L64(rdi, 56)); + r14 = (word64)(r14 ^ WC_L64(rdi, 64)); + r11 = (word64)(r11 ^ WC_L64(rdi, 72)); + r10 = (word64)(r10 ^ WC_L64(rdi, 88)); + r12 = (word64)(r12 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -72)); + r12 = (word64)(WC_L64(rdi, -88)); + r13 = (word64)(WC_L64(rdi, -64)); + r14 = (word64)(WC_L64(rdi, -80)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -64) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -80) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000000000080); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 24)); + r11 = (word64)(WC_L64(rdi, 48)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 56)); + r14 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 56) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 40) = (word64)(r15); + WC_S64(rdi, 24) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -56)); + r11 = (word64)(WC_L64(rdi, -32)); + r12 = (word64)(WC_L64(rdi, -48)); + r13 = (word64)(WC_L64(rdi, -24)); + r14 = (word64)(WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -40) = (word64)(r15); + WC_S64(rdi, -56) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 64)); + r11 = (word64)(WC_L64(rdi, 88)); + r12 = (word64)(WC_L64(rdi, 72)); + r13 = (word64)(WC_L64(rdi, 96)); + r14 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 80) = (word64)(r15); + WC_S64(rdi, 64) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -16)); + r8 = (word64)(r8 ^ WC_L64(rdi, 8)); + r9 = (word64)(r9 ^ WC_L64(rdi, -8)); + rdx = (word64)(rdx ^ WC_L64(rdi, 16)); + rax = (word64)(rax ^ WC_L64(rdi, 0)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -16) = (word64)(r10); + WC_S64(rdi, 8) = (word64)(r11); + WC_S64(rdi, -8) = (word64)(r12); + WC_S64(rdi, 16) = (word64)(r13); + WC_S64(rdi, 0) = (word64)(r14); + /* Round 18 */ + r10 = (word64)(r10 ^ rsi); + r12 = (word64)(r12 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r11 = (word64)(r11 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -40)); + r11 = (word64)(r11 ^ WC_L64(rdi, -32)); + r13 = (word64)(r13 ^ WC_L64(rdi, -24)); + r10 = (word64)(r10 ^ WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 40)); + r11 = (word64)(r11 ^ WC_L64(rdi, 48)); + r13 = (word64)(r13 ^ WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 80)); + r11 = (word64)(r11 ^ WC_L64(rdi, 88)); + r13 = (word64)(r13 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 48)); + r12 = (word64)(WC_L64(rdi, -48)); + r13 = (word64)(WC_L64(rdi, 96)); + r14 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 96) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 0) = (word64)(r13); + /* XOR in constant */ + rsi = (word64)(rsi ^ 0x800a); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -64)); + r11 = (word64)(WC_L64(rdi, 40)); + r12 = (word64)(WC_L64(rdi, -56)); + r13 = (word64)(WC_L64(rdi, 88)); + r14 = (word64)(WC_L64(rdi, -8)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 88) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -8) = (word64)(r15); + WC_S64(rdi, -64) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -72)); + r11 = (word64)(WC_L64(rdi, 32)); + r12 = (word64)(WC_L64(rdi, -24)); + r13 = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(WC_L64(rdi, -16)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -16) = (word64)(r15); + WC_S64(rdi, -72) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, -80)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, -32)); + r13 = (word64)(WC_L64(rdi, 72)); + r14 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 16) = (word64)(r15); + WC_S64(rdi, -80) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -88)); + r8 = (word64)(r8 ^ WC_L64(rdi, 56)); + r9 = (word64)(r9 ^ WC_L64(rdi, -40)); + rdx = (word64)(rdx ^ WC_L64(rdi, 64)); + rax = (word64)(rax ^ WC_L64(rdi, 8)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -88) = (word64)(r10); + WC_S64(rdi, 56) = (word64)(r11); + WC_S64(rdi, -40) = (word64)(r12); + WC_S64(rdi, 64) = (word64)(r13); + WC_S64(rdi, 8) = (word64)(r14); + /* Round 19 */ + r10 = (word64)(r10 ^ rsi); + r10 = (word64)(r10 ^ WC_L64(rdi, -80)); + r10 = (word64)(r10 ^ WC_L64(rdi, -72)); + r10 = (word64)(r10 ^ WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ WC_L64(rdi, -24)); + r14 = (word64)(r14 ^ WC_L64(rdi, -16)); + r14 = (word64)(r14 ^ WC_L64(rdi, -8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 0)); + r14 = (word64)(r14 ^ WC_L64(rdi, 16)); + r11 = (word64)(r11 ^ WC_L64(rdi, 24)); + r11 = (word64)(r11 ^ WC_L64(rdi, 32)); + r11 = (word64)(r11 ^ WC_L64(rdi, 40)); + r11 = (word64)(r11 ^ WC_L64(rdi, 48)); + r13 = (word64)(r13 ^ WC_L64(rdi, 72)); + r13 = (word64)(r13 ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 88)); + r13 = (word64)(r13 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, 40)); + r12 = (word64)(WC_L64(rdi, -24)); + r13 = (word64)(WC_L64(rdi, 72)); + r14 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -24) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 72) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 8) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x800000008000000a); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 96)); + r11 = (word64)(WC_L64(rdi, -8)); + r12 = (word64)(WC_L64(rdi, -72)); + r13 = (word64)(WC_L64(rdi, 24)); + r14 = (word64)(WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 24) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -40) = (word64)(r15); + WC_S64(rdi, 96) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 48)); + r11 = (word64)(WC_L64(rdi, -56)); + r12 = (word64)(WC_L64(rdi, 80)); + r13 = (word64)(WC_L64(rdi, 16)); + r14 = (word64)(WC_L64(rdi, -88)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -88) = (word64)(r15); + WC_S64(rdi, 48) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 0)); + r11 = (word64)(WC_L64(rdi, -64)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, -32)); + r14 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 64) = (word64)(r15); + WC_S64(rdi, 0) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -48)); + r8 = (word64)(r8 ^ WC_L64(rdi, 88)); + r9 = (word64)(r9 ^ WC_L64(rdi, -16)); + rdx = (word64)(rdx ^ WC_L64(rdi, -80)); + rax = (word64)(rax ^ WC_L64(rdi, 56)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -48) = (word64)(r10); + WC_S64(rdi, 88) = (word64)(r11); + WC_S64(rdi, -16) = (word64)(r12); + WC_S64(rdi, -80) = (word64)(r13); + WC_S64(rdi, 56) = (word64)(r14); + /* Round 20 */ + r10 = (word64)(r10 ^ rsi); + r14 = (word64)(r14 ^ WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ WC_L64(rdi, -72)); + r11 = (word64)(r11 ^ WC_L64(rdi, -64)); + r11 = (word64)(r11 ^ WC_L64(rdi, -56)); + r14 = (word64)(r14 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ WC_L64(rdi, -24)); + r11 = (word64)(r11 ^ WC_L64(rdi, -8)); + r10 = (word64)(r10 ^ WC_L64(rdi, 0)); + r14 = (word64)(r14 ^ WC_L64(rdi, 8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 32)); + r11 = (word64)(r11 ^ WC_L64(rdi, 40)); + r10 = (word64)(r10 ^ WC_L64(rdi, 48)); + r14 = (word64)(r14 ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ WC_L64(rdi, 80)); + r10 = (word64)(r10 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -8)); + r12 = (word64)(WC_L64(rdi, 80)); + r13 = (word64)(WC_L64(rdi, -32)); + r14 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 80) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -32) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 56) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000080008081); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 72)); + r11 = (word64)(WC_L64(rdi, -40)); + r12 = (word64)(WC_L64(rdi, 48)); + r13 = (word64)(WC_L64(rdi, -64)); + r14 = (word64)(WC_L64(rdi, -16)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -64) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -16) = (word64)(r15); + WC_S64(rdi, 72) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, 40)); + r11 = (word64)(WC_L64(rdi, -72)); + r12 = (word64)(WC_L64(rdi, 16)); + r13 = (word64)(WC_L64(rdi, 64)); + r14 = (word64)(WC_L64(rdi, -48)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -48) = (word64)(r15); + WC_S64(rdi, 40) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 8)); + r11 = (word64)(WC_L64(rdi, 96)); + r12 = (word64)(WC_L64(rdi, -56)); + r13 = (word64)(WC_L64(rdi, 32)); + r14 = (word64)(WC_L64(rdi, -80)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -80) = (word64)(r15); + WC_S64(rdi, 8) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, -24)); + r8 = (word64)(r8 ^ WC_L64(rdi, 24)); + r9 = (word64)(r9 ^ WC_L64(rdi, -88)); + rdx = (word64)(rdx ^ WC_L64(rdi, 0)); + rax = (word64)(rax ^ WC_L64(rdi, 88)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, -24) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rdi, -88) = (word64)(r12); + WC_S64(rdi, 0) = (word64)(r13); + WC_S64(rdi, 88) = (word64)(r14); + /* Round 21 */ + r10 = (word64)(r10 ^ rsi); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r11 = (word64)(r11 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ WC_L64(rdi, -56)); + r14 = (word64)(r14 ^ WC_L64(rdi, -48)); + r11 = (word64)(r11 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -16)); + r11 = (word64)(r11 ^ WC_L64(rdi, -8)); + r10 = (word64)(r10 ^ WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 32)); + r10 = (word64)(r10 ^ WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ WC_L64(rdi, 48)); + r14 = (word64)(r14 ^ WC_L64(rdi, 56)); + r13 = (word64)(r13 ^ WC_L64(rdi, 64)); + r10 = (word64)(r10 ^ WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ WC_L64(rdi, 80)); + r11 = (word64)(r11 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -40)); + r12 = (word64)(WC_L64(rdi, 16)); + r13 = (word64)(WC_L64(rdi, 32)); + r14 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 16) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 32) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 88) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000000008080); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -32)); + r11 = (word64)(WC_L64(rdi, -16)); + r12 = (word64)(WC_L64(rdi, 40)); + r13 = (word64)(WC_L64(rdi, 96)); + r14 = (word64)(WC_L64(rdi, -88)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 96) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -88) = (word64)(r15); + WC_S64(rdi, -32) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -8)); + r11 = (word64)(WC_L64(rdi, 48)); + r12 = (word64)(WC_L64(rdi, 64)); + r13 = (word64)(WC_L64(rdi, -80)); + r14 = (word64)(WC_L64(rdi, -24)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -24) = (word64)(r15); + WC_S64(rdi, -8) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 56)); + r11 = (word64)(WC_L64(rdi, 72)); + r12 = (word64)(WC_L64(rdi, -72)); + r13 = (word64)(WC_L64(rdi, -56)); + r14 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -56) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 0) = (word64)(r15); + WC_S64(rdi, 56) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 80)); + r8 = (word64)(r8 ^ WC_L64(rdi, -64)); + r9 = (word64)(r9 ^ WC_L64(rdi, -48)); + rdx = (word64)(rdx ^ WC_L64(rdi, 8)); + rax = (word64)(rax ^ WC_L64(rdi, 24)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 80) = (word64)(r10); + WC_S64(rdi, -64) = (word64)(r11); + WC_S64(rdi, -48) = (word64)(r12); + WC_S64(rdi, 8) = (word64)(r13); + WC_S64(rdi, 24) = (word64)(r14); + /* Round 22 */ + r10 = (word64)(r10 ^ rsi); + r14 = (word64)(r14 ^ WC_L64(rdi, -88)); + r13 = (word64)(r13 ^ WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -56)); + r11 = (word64)(r11 ^ WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -24)); + r11 = (word64)(r11 ^ WC_L64(rdi, -16)); + r10 = (word64)(r10 ^ WC_L64(rdi, -8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 32)); + r12 = (word64)(r12 ^ WC_L64(rdi, 40)); + r11 = (word64)(r11 ^ WC_L64(rdi, 48)); + r10 = (word64)(r10 ^ WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ WC_L64(rdi, 64)); + r11 = (word64)(r11 ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 88)); + r13 = (word64)(r13 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -16)); + r12 = (word64)(WC_L64(rdi, 64)); + r13 = (word64)(WC_L64(rdi, -56)); + r14 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -16) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 64) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -56) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, 24) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x80000001); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, 32)); + r11 = (word64)(WC_L64(rdi, -88)); + r12 = (word64)(WC_L64(rdi, -8)); + r13 = (word64)(WC_L64(rdi, 72)); + r14 = (word64)(WC_L64(rdi, -48)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 72) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -48) = (word64)(r15); + WC_S64(rdi, 32) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -40)); + r11 = (word64)(WC_L64(rdi, 40)); + r12 = (word64)(WC_L64(rdi, -80)); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 80) = (word64)(r15); + WC_S64(rdi, -40) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 88)); + r11 = (word64)(WC_L64(rdi, -32)); + r12 = (word64)(WC_L64(rdi, 48)); + r13 = (word64)(WC_L64(rdi, -72)); + r14 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -72) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 8) = (word64)(r15); + WC_S64(rdi, 88) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 16)); + r8 = (word64)(r8 ^ WC_L64(rdi, 96)); + r9 = (word64)(r9 ^ WC_L64(rdi, -24)); + rdx = (word64)(rdx ^ WC_L64(rdi, 56)); + rax = (word64)(rax ^ WC_L64(rdi, -64)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 96) = (word64)(r11); + WC_S64(rdi, -24) = (word64)(r12); + WC_S64(rdi, 56) = (word64)(r13); + WC_S64(rdi, -64) = (word64)(r14); + /* Round 23 */ + r10 = (word64)(r10 ^ rsi); + r11 = (word64)(r11 ^ WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ WC_L64(rdi, -80)); + r13 = (word64)(r13 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -56)); + r14 = (word64)(r14 ^ WC_L64(rdi, -48)); + r10 = (word64)(r10 ^ WC_L64(rdi, -40)); + r11 = (word64)(r11 ^ WC_L64(rdi, -32)); + r11 = (word64)(r11 ^ WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ WC_L64(rdi, -8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 0)); + r14 = (word64)(r14 ^ WC_L64(rdi, 8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 24)); + r10 = (word64)(r10 ^ WC_L64(rdi, 32)); + r11 = (word64)(r11 ^ WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 80)); + r10 = (word64)(r10 ^ WC_L64(rdi, 88)); + /* Calc t[0..4] */ + rdx = (word64)(((r11) >> 63) | ((r11) << 1)); + rax = (word64)(((r12) >> 63) | ((r12) << 1)); + rcx = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r10) >> 63) | ((r10) << 1)); + rdx = (word64)(rdx ^ r14); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx ^ r11); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + /* Row Mix */ + /* Row 0 */ + r10 = (word64)(rsi); + r11 = (word64)(WC_L64(rdi, -88)); + r12 = (word64)(WC_L64(rdi, -80)); + r13 = (word64)(WC_L64(rdi, -72)); + r14 = (word64)(WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r11 = (word64)(((r11) << 44) | ((r11) >> 20)); + r12 = (word64)(((r12) << 43) | ((r12) >> 21)); + r13 = (word64)(((r13) << 21) | ((r13) >> 43)); + r14 = (word64)(((r14) << 14) | ((r14) >> 50)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -88) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -80) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -72) = (word64)(r15); + r13 = (word64)((~r10) & r11); + rsi = (word64)((~r11) & r12); + r13 = (word64)(r13 ^ r14); + rsi = (word64)(rsi ^ r10); + WC_S64(rdi, -64) = (word64)(r13); + /* XOR in constant */ + r14 = (word64)(0x8000000080008008); + rsi = (word64)(rsi ^ r14); + /* Row 1 */ + r10 = (word64)(WC_L64(rdi, -56)); + r11 = (word64)(WC_L64(rdi, -48)); + r12 = (word64)(WC_L64(rdi, -40)); + r13 = (word64)(WC_L64(rdi, -32)); + r14 = (word64)(WC_L64(rdi, -24)); + r10 = (word64)(r10 ^ r8); + r11 = (word64)(r11 ^ r9); + r12 = (word64)(r12 ^ rdx); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ rcx); + r10 = (word64)(((r10) << 28) | ((r10) >> 36)); + r11 = (word64)(((r11) << 20) | ((r11) >> 44)); + r12 = (word64)(((r12) << 3) | ((r12) >> 61)); + r13 = (word64)(((r13) << 45) | ((r13) >> 19)); + r14 = (word64)(((r14) << 61) | ((r14) >> 3)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -48) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -40) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, -32) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, -24) = (word64)(r15); + WC_S64(rdi, -56) = (word64)(r13); + /* Row 2 */ + r10 = (word64)(WC_L64(rdi, -16)); + r11 = (word64)(WC_L64(rdi, -8)); + r12 = (word64)(WC_L64(rdi, 0)); + r13 = (word64)(WC_L64(rdi, 8)); + r14 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(r10 ^ rax); + r11 = (word64)(r11 ^ rcx); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ rdx); + r10 = (word64)(((r10) << 1) | ((r10) >> 63)); + r11 = (word64)(((r11) << 6) | ((r11) >> 58)); + r12 = (word64)(((r12) << 25) | ((r12) >> 39)); + r13 = (word64)(((r13) << 8) | ((r13) >> 56)); + r14 = (word64)(((r14) << 18) | ((r14) >> 46)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, -8) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 0) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 8) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 16) = (word64)(r15); + WC_S64(rdi, -16) = (word64)(r13); + /* Row 3 */ + r10 = (word64)(WC_L64(rdi, 24)); + r11 = (word64)(WC_L64(rdi, 32)); + r12 = (word64)(WC_L64(rdi, 40)); + r13 = (word64)(WC_L64(rdi, 48)); + r14 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ r9); + r11 = (word64)(r11 ^ rdx); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ rcx); + r14 = (word64)(r14 ^ r8); + r10 = (word64)(((r10) << 27) | ((r10) >> 37)); + r11 = (word64)(((r11) << 36) | ((r11) >> 28)); + r12 = (word64)(((r12) << 10) | ((r12) >> 54)); + r13 = (word64)(((r13) << 15) | ((r13) >> 49)); + r14 = (word64)(((r14) << 56) | ((r14) >> 8)); + r15 = (word64)((~r12) & r13); + r15 = (word64)(r15 ^ r11); + WC_S64(rdi, 32) = (word64)(r15); + r15 = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 40) = (word64)(r15); + r15 = (word64)((~r14) & r10); + r15 = (word64)(r15 ^ r13); + WC_S64(rdi, 48) = (word64)(r15); + r15 = (word64)((~r10) & r11); + r13 = (word64)((~r11) & r12); + r15 = (word64)(r15 ^ r14); + r13 = (word64)(r13 ^ r10); + WC_S64(rdi, 56) = (word64)(r15); + WC_S64(rdi, 24) = (word64)(r13); + /* Row 4 */ + rcx = (word64)(rcx ^ WC_L64(rdi, 64)); + r8 = (word64)(r8 ^ WC_L64(rdi, 72)); + r9 = (word64)(r9 ^ WC_L64(rdi, 80)); + rdx = (word64)(rdx ^ WC_L64(rdi, 88)); + rax = (word64)(rax ^ WC_L64(rdi, 96)); + r10 = (word64)(((rcx) >> 2) | ((rcx) << 62)); + r11 = (word64)(((r8) >> 9) | ((r8) << 55)); + r12 = (word64)(((r9) >> 25) | ((r9) << 39)); + r13 = (word64)(((rdx) >> 23) | ((rdx) << 41)); + r14 = (word64)(((rax) >> 62) | ((rax) << 2)); + rdx = (word64)((~r11) & r12); + rax = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r10); + r9 = (word64)((~r10) & r11); + r10 = (word64)(r10 ^ rdx); + r11 = (word64)(r11 ^ rax); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + WC_S64(rdi, 64) = (word64)(r10); + WC_S64(rdi, 72) = (word64)(r11); + WC_S64(rdi, 80) = (word64)(r12); + WC_S64(rdi, 88) = (word64)(r13); + WC_S64(rdi, 96) = (word64)(r14); + WC_S64(rdi, -96) = (word64)(rsi); +} + +WC_X64I_TARGET("bmi,bmi2") +WOLFSSL_LOCAL void sha3_block_n_bmi2(word64* s, const byte* data, word32 n, + word64 c) +{ + word64 rdi, rsi, rdx, rcx, rsp, rbp, r12 = 0, r13 = 0, r14 = 0, r15 = 0, + rbx = 0, rax = 0, r8 = 0, r9 = 0, r10 = 0, r11 = 0; + XALIGNED(32) WC_X64I_SLOT stk[4]; + word32 zf1; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)data; + rdx = (word64)(word32)n; + rcx = (word64)(word64)c; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rcx; + rbp = (word64)(rcx); + rcx = (word64)(WC_L64(rdi, 0)); + rdi = (word64)(rdi + 0x60); +L_sha3_block_n_bmi2_start: + if ((rbp) == (0x88)) { + goto L_sha3_block_n_bmi2_load_256; + } + if ((rbp) == (0xa8)) { + goto L_sha3_block_n_bmi2_load_128; + } + if ((rbp) == (0x90)) { + goto L_sha3_block_n_bmi2_load_224; + } + if ((rbp) == (0x68)) { + goto L_sha3_block_n_bmi2_load_384; + } + r12 = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(WC_L64(rsi, 8)); + r14 = (word64)(WC_L64(rsi, 16)); + r15 = (word64)(WC_L64(rsi, 24)); + rbx = (word64)(WC_L64(rsi, 32)); + rax = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)(WC_L64(rsi, 64)); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r15 = (word64)(r15 ^ WC_L64(rdi, -72)); + rbx = (word64)(rbx ^ WC_L64(rdi, -64)); + rax = (word64)(rax ^ WC_L64(rdi, -56)); + r8 = (word64)(r8 ^ WC_L64(rdi, -48)); + r9 = (word64)(r9 ^ WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ WC_L64(rdi, -32)); + rcx = (word64)(r12); + WC_S64(rdi, -88) = (word64)(r13); + WC_S64(rdi, -80) = (word64)(r14); + WC_S64(rdi, -72) = (word64)(r15); + WC_S64(rdi, -64) = (word64)(rbx); + WC_S64(rdi, -56) = (word64)(rax); + WC_S64(rdi, -48) = (word64)(r8); + WC_S64(rdi, -40) = (word64)(r9); + WC_S64(rdi, -32) = (word64)(r10); + goto L_sha3_block_n_bmi2_rounds; +L_sha3_block_n_bmi2_load_128: + r12 = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(WC_L64(rsi, 8)); + r14 = (word64)(WC_L64(rsi, 16)); + r15 = (word64)(WC_L64(rsi, 24)); + rbx = (word64)(WC_L64(rsi, 32)); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r15 = (word64)(r15 ^ WC_L64(rdi, -72)); + rbx = (word64)(rbx ^ WC_L64(rdi, -64)); + rcx = (word64)(r12); + WC_S64(rdi, -88) = (word64)(r13); + WC_S64(rdi, -80) = (word64)(r14); + WC_S64(rdi, -72) = (word64)(r15); + WC_S64(rdi, -64) = (word64)(rbx); + rax = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)(WC_L64(rsi, 64)); + r11 = (word64)(WC_L64(rsi, 72)); + rbp = (word64)(WC_L64(rsi, 80)); + rax = (word64)(rax ^ WC_L64(rdi, -56)); + r8 = (word64)(r8 ^ WC_L64(rdi, -48)); + r9 = (word64)(r9 ^ WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ WC_L64(rdi, -32)); + r11 = (word64)(r11 ^ WC_L64(rdi, -24)); + rbp = (word64)(rbp ^ WC_L64(rdi, -16)); + WC_S64(rdi, -56) = (word64)(rax); + WC_S64(rdi, -48) = (word64)(r8); + WC_S64(rdi, -40) = (word64)(r9); + WC_S64(rdi, -32) = (word64)(r10); + WC_S64(rdi, -24) = (word64)(r11); + WC_S64(rdi, -16) = (word64)(rbp); + rax = (word64)(WC_L64(rsi, 88)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)(WC_L64(rsi, 112)); + r11 = (word64)(WC_L64(rsi, 120)); + rbp = (word64)(WC_L64(rsi, 128)); + rax = (word64)(rax ^ WC_L64(rdi, -8)); + r8 = (word64)(r8 ^ WC_L64(rdi, 0)); + r9 = (word64)(r9 ^ WC_L64(rdi, 8)); + r10 = (word64)(r10 ^ WC_L64(rdi, 16)); + r11 = (word64)(r11 ^ WC_L64(rdi, 24)); + rbp = (word64)(rbp ^ WC_L64(rdi, 32)); + WC_S64(rdi, -8) = (word64)(rax); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rdi, 32) = (word64)(rbp); + rax = (word64)(WC_L64(rsi, 136)); + r8 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)(WC_L64(rsi, 152)); + r10 = (word64)(WC_L64(rsi, 160)); + rax = (word64)(rax ^ WC_L64(rdi, 40)); + r8 = (word64)(r8 ^ WC_L64(rdi, 48)); + r9 = (word64)(r9 ^ WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ WC_L64(rdi, 64)); + WC_S64(rdi, 40) = (word64)(rax); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + WC_S64(rdi, 64) = (word64)(r10); + goto L_sha3_block_n_bmi2_rounds; +L_sha3_block_n_bmi2_load_224: + r12 = (word64)(WC_L64(rsi, 40)); + r13 = (word64)(WC_L64(rsi, 48)); + r14 = (word64)(WC_L64(rsi, 56)); + r15 = (word64)(WC_L64(rsi, 64)); + rbx = (word64)(WC_L64(rsi, 72)); + rax = (word64)(WC_L64(rsi, 80)); + r8 = (word64)(WC_L64(rsi, 88)); + r9 = (word64)(WC_L64(rsi, 96)); + r10 = (word64)(WC_L64(rsi, 104)); + r11 = (word64)(WC_L64(rsi, 112)); + r12 = (word64)(r12 ^ WC_L64(rdi, -56)); + r13 = (word64)(r13 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -40)); + r15 = (word64)(r15 ^ WC_L64(rdi, -32)); + rbx = (word64)(rbx ^ WC_L64(rdi, -24)); + rax = (word64)(rax ^ WC_L64(rdi, -16)); + r8 = (word64)(r8 ^ WC_L64(rdi, -8)); + r9 = (word64)(r9 ^ WC_L64(rdi, 0)); + r10 = (word64)(r10 ^ WC_L64(rdi, 8)); + r11 = (word64)(r11 ^ WC_L64(rdi, 16)); + WC_S64(rdi, -56) = (word64)(r12); + WC_S64(rdi, -48) = (word64)(r13); + WC_S64(rdi, -40) = (word64)(r14); + WC_S64(rdi, -32) = (word64)(r15); + WC_S64(rdi, -24) = (word64)(rbx); + WC_S64(rdi, -16) = (word64)(rax); + WC_S64(rdi, -8) = (word64)(r8); + WC_S64(rdi, 0) = (word64)(r9); + WC_S64(rdi, 8) = (word64)(r10); + WC_S64(rdi, 16) = (word64)(r11); + r12 = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(WC_L64(rsi, 8)); + r14 = (word64)(WC_L64(rsi, 16)); + r15 = (word64)(WC_L64(rsi, 24)); + rbx = (word64)(WC_L64(rsi, 32)); + rax = (word64)(WC_L64(rsi, 120)); + r8 = (word64)(WC_L64(rsi, 128)); + r9 = (word64)(WC_L64(rsi, 136)); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r15 = (word64)(r15 ^ WC_L64(rdi, -72)); + rbx = (word64)(rbx ^ WC_L64(rdi, -64)); + rax = (word64)(rax ^ WC_L64(rdi, 24)); + r8 = (word64)(r8 ^ WC_L64(rdi, 32)); + r9 = (word64)(r9 ^ WC_L64(rdi, 40)); + rcx = (word64)(r12); + WC_S64(rdi, -88) = (word64)(r13); + WC_S64(rdi, -80) = (word64)(r14); + WC_S64(rdi, -72) = (word64)(r15); + WC_S64(rdi, -64) = (word64)(rbx); + WC_S64(rdi, 24) = (word64)(rax); + WC_S64(rdi, 32) = (word64)(r8); + WC_S64(rdi, 40) = (word64)(r9); + goto L_sha3_block_n_bmi2_rounds; +L_sha3_block_n_bmi2_load_384: + r12 = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(WC_L64(rsi, 8)); + r14 = (word64)(WC_L64(rsi, 16)); + r15 = (word64)(WC_L64(rsi, 24)); + rbx = (word64)(WC_L64(rsi, 32)); + rax = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)(WC_L64(rsi, 64)); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r15 = (word64)(r15 ^ WC_L64(rdi, -72)); + rbx = (word64)(rbx ^ WC_L64(rdi, -64)); + rax = (word64)(rax ^ WC_L64(rdi, -56)); + r8 = (word64)(r8 ^ WC_L64(rdi, -48)); + r9 = (word64)(r9 ^ WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ WC_L64(rdi, -32)); + rcx = (word64)(r12); + WC_S64(rdi, -88) = (word64)(r13); + WC_S64(rdi, -80) = (word64)(r14); + WC_S64(rdi, -72) = (word64)(r15); + WC_S64(rdi, -64) = (word64)(rbx); + WC_S64(rdi, -56) = (word64)(rax); + WC_S64(rdi, -48) = (word64)(r8); + WC_S64(rdi, -40) = (word64)(r9); + WC_S64(rdi, -32) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 72)); + r8 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(WC_L64(rsi, 88)); + r10 = (word64)(WC_L64(rsi, 96)); + rax = (word64)(rax ^ WC_L64(rdi, -24)); + r8 = (word64)(r8 ^ WC_L64(rdi, -16)); + r9 = (word64)(r9 ^ WC_L64(rdi, -8)); + r10 = (word64)(r10 ^ WC_L64(rdi, 0)); + WC_S64(rdi, -24) = (word64)(rax); + WC_S64(rdi, -16) = (word64)(r8); + WC_S64(rdi, -8) = (word64)(r9); + WC_S64(rdi, 0) = (word64)(r10); + goto L_sha3_block_n_bmi2_rounds; +L_sha3_block_n_bmi2_load_256: + r12 = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(WC_L64(rsi, 8)); + r14 = (word64)(WC_L64(rsi, 16)); + r15 = (word64)(WC_L64(rsi, 24)); + rbx = (word64)(WC_L64(rsi, 32)); + rax = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)(WC_L64(rsi, 64)); + r11 = (word64)(WC_L64(rsi, 72)); + rbp = (word64)(WC_L64(rsi, 80)); + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r15 = (word64)(r15 ^ WC_L64(rdi, -72)); + rbx = (word64)(rbx ^ WC_L64(rdi, -64)); + rax = (word64)(rax ^ WC_L64(rdi, -56)); + r8 = (word64)(r8 ^ WC_L64(rdi, -48)); + r9 = (word64)(r9 ^ WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ WC_L64(rdi, -32)); + r11 = (word64)(r11 ^ WC_L64(rdi, -24)); + rbp = (word64)(rbp ^ WC_L64(rdi, -16)); + rcx = (word64)(r12); + WC_S64(rdi, -88) = (word64)(r13); + WC_S64(rdi, -80) = (word64)(r14); + WC_S64(rdi, -72) = (word64)(r15); + WC_S64(rdi, -64) = (word64)(rbx); + WC_S64(rdi, -56) = (word64)(rax); + WC_S64(rdi, -48) = (word64)(r8); + WC_S64(rdi, -40) = (word64)(r9); + WC_S64(rdi, -32) = (word64)(r10); + WC_S64(rdi, -24) = (word64)(r11); + WC_S64(rdi, -16) = (word64)(rbp); + rax = (word64)(WC_L64(rsi, 88)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)(WC_L64(rsi, 112)); + r11 = (word64)(WC_L64(rsi, 120)); + rbp = (word64)(WC_L64(rsi, 128)); + rax = (word64)(rax ^ WC_L64(rdi, -8)); + r8 = (word64)(r8 ^ WC_L64(rdi, 0)); + r9 = (word64)(r9 ^ WC_L64(rdi, 8)); + r10 = (word64)(r10 ^ WC_L64(rdi, 16)); + r11 = (word64)(r11 ^ WC_L64(rdi, 24)); + rbp = (word64)(rbp ^ WC_L64(rdi, 32)); + WC_S64(rdi, -8) = (word64)(rax); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rdi, 32) = (word64)(rbp); +L_sha3_block_n_bmi2_rounds: + /* Round 0 */ + r12 = (word64)(r12 ^ WC_L64(rdi, -56)); + r13 = (word64)(r13 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -40)); + r15 = (word64)(r15 ^ WC_L64(rdi, -32)); + rbx = (word64)(rbx ^ WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ WC_L64(rdi, -16)); + r13 = (word64)(r13 ^ WC_L64(rdi, -8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 0)); + r15 = (word64)(r15 ^ WC_L64(rdi, 8)); + rbx = (word64)(rbx ^ WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ WC_L64(rdi, 24)); + r13 = (word64)(r13 ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 40)); + r15 = (word64)(r15 ^ WC_L64(rdi, 48)); + rbx = (word64)(rbx ^ WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 80)); + r15 = (word64)(r15 ^ WC_L64(rdi, 88)); + rbx = (word64)(rbx ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -48)); + r14 = (word64)(WC_L64(rdi, 0)); + r15 = (word64)(WC_L64(rdi, 48)); + rbx = (word64)(WC_L64(rdi, 96)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 48) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 96) = (word64)(r15); + /* XOR in constant */ + rcx = (word64)(rcx ^ 1); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -72)); + r13 = (word64)(WC_L64(rdi, -24)); + r14 = (word64)(WC_L64(rdi, -16)); + r15 = (word64)(WC_L64(rdi, 32)); + rbx = (word64)(WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 80) = (word64)(rbp); + WC_S64(rdi, -72) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -88)); + r13 = (word64)(WC_L64(rdi, -40)); + r14 = (word64)(WC_L64(rdi, 8)); + r15 = (word64)(WC_L64(rdi, 56)); + rbx = (word64)(WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 64) = (word64)(rbp); + WC_S64(rdi, -88) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -64)); + r13 = (word64)(WC_L64(rdi, -56)); + r14 = (word64)(WC_L64(rdi, -8)); + r15 = (word64)(WC_L64(rdi, 40)); + rbx = (word64)(WC_L64(rdi, 88)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 88) = (word64)(rbp); + WC_S64(rdi, -64) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -80)); + r10 = (word64)(r10 ^ WC_L64(rdi, -32)); + r11 = (word64)(r11 ^ WC_L64(rdi, 16)); + rax = (word64)(rax ^ WC_L64(rdi, 24)); + r8 = (word64)(r8 ^ WC_L64(rdi, 72)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -80) = (word64)(r12); + WC_S64(rdi, -32) = (word64)(r13); + WC_S64(rdi, 16) = (word64)(r14); + WC_S64(rdi, 24) = (word64)(r15); + WC_S64(rdi, 72) = (word64)(rbx); + /* Round 1 */ + r12 = (word64)(r12 ^ rcx); + r12 = (word64)(r12 ^ WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ WC_L64(rdi, -72)); + r12 = (word64)(r12 ^ WC_L64(rdi, -64)); + r13 = (word64)(r13 ^ WC_L64(rdi, -56)); + r13 = (word64)(r13 ^ WC_L64(rdi, -48)); + r13 = (word64)(r13 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -24)); + r14 = (word64)(r14 ^ WC_L64(rdi, -16)); + r14 = (word64)(r14 ^ WC_L64(rdi, -8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 0)); + r14 = (word64)(r14 ^ WC_L64(rdi, 8)); + r15 = (word64)(r15 ^ WC_L64(rdi, 32)); + r15 = (word64)(r15 ^ WC_L64(rdi, 40)); + r15 = (word64)(r15 ^ WC_L64(rdi, 48)); + r15 = (word64)(r15 ^ WC_L64(rdi, 56)); + rbx = (word64)(rbx ^ WC_L64(rdi, 64)); + rbx = (word64)(rbx ^ WC_L64(rdi, 80)); + rbx = (word64)(rbx ^ WC_L64(rdi, 88)); + rbx = (word64)(rbx ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -24)); + r14 = (word64)(WC_L64(rdi, 8)); + r15 = (word64)(WC_L64(rdi, 40)); + rbx = (word64)(WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 40) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 72) = (word64)(r15); + /* XOR in constant */ + rcx = (word64)(rcx ^ 0x8082); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 48)); + r13 = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(WC_L64(rdi, -88)); + r15 = (word64)(WC_L64(rdi, -56)); + rbx = (word64)(WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 16) = (word64)(rbp); + WC_S64(rdi, 48) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -48)); + r13 = (word64)(WC_L64(rdi, -16)); + r14 = (word64)(WC_L64(rdi, 56)); + r15 = (word64)(WC_L64(rdi, 88)); + rbx = (word64)(WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -80) = (word64)(rbp); + WC_S64(rdi, -48) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 96)); + r13 = (word64)(WC_L64(rdi, -72)); + r14 = (word64)(WC_L64(rdi, -40)); + r15 = (word64)(WC_L64(rdi, -8)); + rbx = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 24) = (word64)(rbp); + WC_S64(rdi, 96) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 0)); + r10 = (word64)(r10 ^ WC_L64(rdi, 32)); + r11 = (word64)(r11 ^ WC_L64(rdi, 64)); + rax = (word64)(rax ^ WC_L64(rdi, -64)); + r8 = (word64)(r8 ^ WC_L64(rdi, -32)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 0) = (word64)(r12); + WC_S64(rdi, 32) = (word64)(r13); + WC_S64(rdi, 64) = (word64)(r14); + WC_S64(rdi, -64) = (word64)(r15); + WC_S64(rdi, -32) = (word64)(rbx); + /* Round 2 */ + r12 = (word64)(r12 ^ rcx); + r14 = (word64)(r14 ^ WC_L64(rdi, -88)); + rbx = (word64)(rbx ^ WC_L64(rdi, -80)); + r13 = (word64)(r13 ^ WC_L64(rdi, -72)); + r15 = (word64)(r15 ^ WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -24)); + r13 = (word64)(r13 ^ WC_L64(rdi, -16)); + r15 = (word64)(r15 ^ WC_L64(rdi, -8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 8)); + rbx = (word64)(rbx ^ WC_L64(rdi, 16)); + rbx = (word64)(rbx ^ WC_L64(rdi, 24)); + r15 = (word64)(r15 ^ WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ WC_L64(rdi, 48)); + r14 = (word64)(r14 ^ WC_L64(rdi, 56)); + rbx = (word64)(rbx ^ WC_L64(rdi, 72)); + r13 = (word64)(r13 ^ WC_L64(rdi, 80)); + r15 = (word64)(r15 ^ WC_L64(rdi, 88)); + r12 = (word64)(r12 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(WC_L64(rdi, 56)); + r15 = (word64)(WC_L64(rdi, -8)); + rbx = (word64)(WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -8) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -32) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x800000000000808a); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 40)); + r13 = (word64)(WC_L64(rdi, 16)); + r14 = (word64)(WC_L64(rdi, -48)); + r15 = (word64)(WC_L64(rdi, -72)); + rbx = (word64)(WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 64) = (word64)(rbp); + WC_S64(rdi, 40) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -24)); + r13 = (word64)(WC_L64(rdi, -88)); + r14 = (word64)(WC_L64(rdi, 88)); + r15 = (word64)(WC_L64(rdi, 24)); + rbx = (word64)(WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 0) = (word64)(rbp); + WC_S64(rdi, -24) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 72)); + r13 = (word64)(WC_L64(rdi, 48)); + r14 = (word64)(WC_L64(rdi, -16)); + r15 = (word64)(WC_L64(rdi, -40)); + rbx = (word64)(WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -64) = (word64)(rbp); + WC_S64(rdi, 72) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 8)); + r10 = (word64)(r10 ^ WC_L64(rdi, -56)); + r11 = (word64)(r11 ^ WC_L64(rdi, -80)); + rax = (word64)(rax ^ WC_L64(rdi, 96)); + r8 = (word64)(r8 ^ WC_L64(rdi, 32)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 8) = (word64)(r12); + WC_S64(rdi, -56) = (word64)(r13); + WC_S64(rdi, -80) = (word64)(r14); + WC_S64(rdi, 96) = (word64)(r15); + WC_S64(rdi, 32) = (word64)(rbx); + /* Round 3 */ + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r15 = (word64)(r15 ^ WC_L64(rdi, -72)); + rbx = (word64)(rbx ^ WC_L64(rdi, -64)); + r14 = (word64)(r14 ^ WC_L64(rdi, -48)); + r15 = (word64)(r15 ^ WC_L64(rdi, -40)); + rbx = (word64)(rbx ^ WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ WC_L64(rdi, -24)); + r14 = (word64)(r14 ^ WC_L64(rdi, -16)); + r15 = (word64)(r15 ^ WC_L64(rdi, -8)); + rbx = (word64)(rbx ^ WC_L64(rdi, 0)); + r13 = (word64)(r13 ^ WC_L64(rdi, 16)); + r15 = (word64)(r15 ^ WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 48)); + r14 = (word64)(r14 ^ WC_L64(rdi, 56)); + rbx = (word64)(rbx ^ WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ WC_L64(rdi, 72)); + r13 = (word64)(r13 ^ WC_L64(rdi, 80)); + r14 = (word64)(r14 ^ WC_L64(rdi, 88)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 16)); + r14 = (word64)(WC_L64(rdi, 88)); + r15 = (word64)(WC_L64(rdi, -40)); + rbx = (word64)(WC_L64(rdi, 32)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -40) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 32) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000080008000); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -8)); + r13 = (word64)(WC_L64(rdi, 64)); + r14 = (word64)(WC_L64(rdi, -24)); + r15 = (word64)(WC_L64(rdi, 48)); + rbx = (word64)(WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -80) = (word64)(rbp); + WC_S64(rdi, -8) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 80)); + r13 = (word64)(WC_L64(rdi, -48)); + r14 = (word64)(WC_L64(rdi, 24)); + r15 = (word64)(WC_L64(rdi, -64)); + rbx = (word64)(WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 8) = (word64)(rbp); + WC_S64(rdi, 80) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, -88)); + r15 = (word64)(WC_L64(rdi, -16)); + rbx = (word64)(WC_L64(rdi, 96)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 96) = (word64)(rbp); + WC_S64(rdi, -32) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 56)); + r10 = (word64)(r10 ^ WC_L64(rdi, -72)); + r11 = (word64)(r11 ^ WC_L64(rdi, 0)); + rax = (word64)(rax ^ WC_L64(rdi, 72)); + r8 = (word64)(r8 ^ WC_L64(rdi, -56)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 56) = (word64)(r12); + WC_S64(rdi, -72) = (word64)(r13); + WC_S64(rdi, 0) = (word64)(r14); + WC_S64(rdi, 72) = (word64)(r15); + WC_S64(rdi, -56) = (word64)(rbx); + /* Round 4 */ + r12 = (word64)(r12 ^ rcx); + r14 = (word64)(r14 ^ WC_L64(rdi, -88)); + rbx = (word64)(rbx ^ WC_L64(rdi, -80)); + r15 = (word64)(r15 ^ WC_L64(rdi, -64)); + r13 = (word64)(r13 ^ WC_L64(rdi, -48)); + r15 = (word64)(r15 ^ WC_L64(rdi, -40)); + r12 = (word64)(r12 ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -24)); + r15 = (word64)(r15 ^ WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ WC_L64(rdi, -8)); + rbx = (word64)(rbx ^ WC_L64(rdi, 8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 16)); + r14 = (word64)(r14 ^ WC_L64(rdi, 24)); + rbx = (word64)(rbx ^ WC_L64(rdi, 32)); + r13 = (word64)(r13 ^ WC_L64(rdi, 40)); + r15 = (word64)(r15 ^ WC_L64(rdi, 48)); + r13 = (word64)(r13 ^ WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ WC_L64(rdi, 80)); + r14 = (word64)(r14 ^ WC_L64(rdi, 88)); + rbx = (word64)(rbx ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 64)); + r14 = (word64)(WC_L64(rdi, 24)); + r15 = (word64)(WC_L64(rdi, -16)); + rbx = (word64)(WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -16) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -56) = (word64)(r15); + /* XOR in constant */ + rcx = (word64)(rcx ^ 0x808b); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -40)); + r13 = (word64)(WC_L64(rdi, -80)); + r14 = (word64)(WC_L64(rdi, 80)); + r15 = (word64)(WC_L64(rdi, 40)); + rbx = (word64)(WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 0) = (word64)(rbp); + WC_S64(rdi, -40) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 16)); + r13 = (word64)(WC_L64(rdi, -24)); + r14 = (word64)(WC_L64(rdi, -64)); + r15 = (word64)(WC_L64(rdi, 96)); + rbx = (word64)(WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 56) = (word64)(rbp); + WC_S64(rdi, 16) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, -8)); + r14 = (word64)(WC_L64(rdi, -48)); + r15 = (word64)(WC_L64(rdi, -88)); + rbx = (word64)(WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 72) = (word64)(rbp); + WC_S64(rdi, 32) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 88)); + r10 = (word64)(r10 ^ WC_L64(rdi, 48)); + r11 = (word64)(r11 ^ WC_L64(rdi, 8)); + rax = (word64)(rax ^ WC_L64(rdi, -32)); + r8 = (word64)(r8 ^ WC_L64(rdi, -72)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 88) = (word64)(r12); + WC_S64(rdi, 48) = (word64)(r13); + WC_S64(rdi, 8) = (word64)(r14); + WC_S64(rdi, -32) = (word64)(r15); + WC_S64(rdi, -72) = (word64)(rbx); + /* Round 5 */ + r12 = (word64)(r12 ^ rcx); + r15 = (word64)(r15 ^ WC_L64(rdi, -88)); + r13 = (word64)(r13 ^ WC_L64(rdi, -80)); + r14 = (word64)(r14 ^ WC_L64(rdi, -64)); + rbx = (word64)(rbx ^ WC_L64(rdi, -56)); + r14 = (word64)(r14 ^ WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -24)); + r15 = (word64)(r15 ^ WC_L64(rdi, -16)); + r13 = (word64)(r13 ^ WC_L64(rdi, -8)); + rbx = (word64)(rbx ^ WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ WC_L64(rdi, 16)); + r14 = (word64)(r14 ^ WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 32)); + r15 = (word64)(r15 ^ WC_L64(rdi, 40)); + rbx = (word64)(rbx ^ WC_L64(rdi, 56)); + r13 = (word64)(r13 ^ WC_L64(rdi, 64)); + rbx = (word64)(rbx ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 80)); + r15 = (word64)(r15 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -80)); + r14 = (word64)(WC_L64(rdi, -64)); + r15 = (word64)(WC_L64(rdi, -88)); + rbx = (word64)(WC_L64(rdi, -72)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -88) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -72) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x80000001); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -16)); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 16)); + r15 = (word64)(WC_L64(rdi, -8)); + rbx = (word64)(WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 8) = (word64)(rbp); + WC_S64(rdi, -16) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 64)); + r13 = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(WC_L64(rdi, 96)); + r15 = (word64)(WC_L64(rdi, 72)); + rbx = (word64)(WC_L64(rdi, 88)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 88) = (word64)(rbp); + WC_S64(rdi, 64) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -56)); + r13 = (word64)(WC_L64(rdi, -40)); + r14 = (word64)(WC_L64(rdi, -24)); + r15 = (word64)(WC_L64(rdi, -48)); + rbx = (word64)(WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -32) = (word64)(rbp); + WC_S64(rdi, -56) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 24)); + r10 = (word64)(r10 ^ WC_L64(rdi, 40)); + r11 = (word64)(r11 ^ WC_L64(rdi, 56)); + rax = (word64)(rax ^ WC_L64(rdi, 32)); + r8 = (word64)(r8 ^ WC_L64(rdi, 48)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 24) = (word64)(r12); + WC_S64(rdi, 40) = (word64)(r13); + WC_S64(rdi, 56) = (word64)(r14); + WC_S64(rdi, 32) = (word64)(r15); + WC_S64(rdi, 48) = (word64)(rbx); + /* Round 6 */ + r12 = (word64)(r12 ^ rcx); + r15 = (word64)(r15 ^ WC_L64(rdi, -88)); + r13 = (word64)(r13 ^ WC_L64(rdi, -80)); + rbx = (word64)(rbx ^ WC_L64(rdi, -72)); + r14 = (word64)(r14 ^ WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ WC_L64(rdi, -56)); + r15 = (word64)(r15 ^ WC_L64(rdi, -48)); + r13 = (word64)(r13 ^ WC_L64(rdi, -40)); + rbx = (word64)(rbx ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ WC_L64(rdi, -16)); + r15 = (word64)(r15 ^ WC_L64(rdi, -8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 0)); + rbx = (word64)(rbx ^ WC_L64(rdi, 8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ WC_L64(rdi, 64)); + r15 = (word64)(r15 ^ WC_L64(rdi, 72)); + r13 = (word64)(r13 ^ WC_L64(rdi, 80)); + rbx = (word64)(rbx ^ WC_L64(rdi, 88)); + r14 = (word64)(r14 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 96)); + r15 = (word64)(WC_L64(rdi, -48)); + rbx = (word64)(WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -48) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 48) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000080008081); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -88)); + r13 = (word64)(WC_L64(rdi, 8)); + r14 = (word64)(WC_L64(rdi, 64)); + r15 = (word64)(WC_L64(rdi, -40)); + rbx = (word64)(WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 56) = (word64)(rbp); + WC_S64(rdi, -88) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -80)); + r13 = (word64)(WC_L64(rdi, 16)); + r14 = (word64)(WC_L64(rdi, 72)); + r15 = (word64)(WC_L64(rdi, -32)); + rbx = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 24) = (word64)(rbp); + WC_S64(rdi, -80) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -72)); + r13 = (word64)(WC_L64(rdi, -16)); + r14 = (word64)(WC_L64(rdi, 80)); + r15 = (word64)(WC_L64(rdi, -24)); + rbx = (word64)(WC_L64(rdi, 32)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 32) = (word64)(rbp); + WC_S64(rdi, -72) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -64)); + r10 = (word64)(r10 ^ WC_L64(rdi, -8)); + r11 = (word64)(r11 ^ WC_L64(rdi, 88)); + rax = (word64)(rax ^ WC_L64(rdi, -56)); + r8 = (word64)(r8 ^ WC_L64(rdi, 40)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -64) = (word64)(r12); + WC_S64(rdi, -8) = (word64)(r13); + WC_S64(rdi, 88) = (word64)(r14); + WC_S64(rdi, -56) = (word64)(r15); + WC_S64(rdi, 40) = (word64)(rbx); + /* Round 7 */ + r12 = (word64)(r12 ^ rcx); + r12 = (word64)(r12 ^ WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ WC_L64(rdi, -72)); + r15 = (word64)(r15 ^ WC_L64(rdi, -48)); + r15 = (word64)(r15 ^ WC_L64(rdi, -40)); + r15 = (word64)(r15 ^ WC_L64(rdi, -32)); + r15 = (word64)(r15 ^ WC_L64(rdi, -24)); + r13 = (word64)(r13 ^ WC_L64(rdi, -16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 0)); + r13 = (word64)(r13 ^ WC_L64(rdi, 8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 16)); + rbx = (word64)(rbx ^ WC_L64(rdi, 24)); + rbx = (word64)(rbx ^ WC_L64(rdi, 32)); + rbx = (word64)(rbx ^ WC_L64(rdi, 48)); + rbx = (word64)(rbx ^ WC_L64(rdi, 56)); + r14 = (word64)(r14 ^ WC_L64(rdi, 64)); + r14 = (word64)(r14 ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 80)); + r14 = (word64)(r14 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 8)); + r14 = (word64)(WC_L64(rdi, 72)); + r15 = (word64)(WC_L64(rdi, -24)); + rbx = (word64)(WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -24) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 40) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000000008009); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -48)); + r13 = (word64)(WC_L64(rdi, 56)); + r14 = (word64)(WC_L64(rdi, -80)); + r15 = (word64)(WC_L64(rdi, -16)); + rbx = (word64)(WC_L64(rdi, 88)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 88) = (word64)(rbp); + WC_S64(rdi, -48) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 0)); + r13 = (word64)(WC_L64(rdi, 64)); + r14 = (word64)(WC_L64(rdi, -32)); + r15 = (word64)(WC_L64(rdi, 32)); + rbx = (word64)(WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -64) = (word64)(rbp); + WC_S64(rdi, 0) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 48)); + r13 = (word64)(WC_L64(rdi, -88)); + r14 = (word64)(WC_L64(rdi, 16)); + r15 = (word64)(WC_L64(rdi, 80)); + rbx = (word64)(WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -56) = (word64)(rbp); + WC_S64(rdi, 48) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 96)); + r10 = (word64)(r10 ^ WC_L64(rdi, -40)); + r11 = (word64)(r11 ^ WC_L64(rdi, 24)); + rax = (word64)(rax ^ WC_L64(rdi, -72)); + r8 = (word64)(r8 ^ WC_L64(rdi, -8)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 96) = (word64)(r12); + WC_S64(rdi, -40) = (word64)(r13); + WC_S64(rdi, 24) = (word64)(r14); + WC_S64(rdi, -72) = (word64)(r15); + WC_S64(rdi, -8) = (word64)(rbx); + /* Round 8 */ + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + rbx = (word64)(rbx ^ WC_L64(rdi, -64)); + rbx = (word64)(rbx ^ WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -32)); + r15 = (word64)(r15 ^ WC_L64(rdi, -24)); + r15 = (word64)(r15 ^ WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ WC_L64(rdi, 0)); + r13 = (word64)(r13 ^ WC_L64(rdi, 8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 16)); + r15 = (word64)(r15 ^ WC_L64(rdi, 32)); + rbx = (word64)(rbx ^ WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ WC_L64(rdi, 48)); + r13 = (word64)(r13 ^ WC_L64(rdi, 56)); + r13 = (word64)(r13 ^ WC_L64(rdi, 64)); + r14 = (word64)(r14 ^ WC_L64(rdi, 72)); + r15 = (word64)(r15 ^ WC_L64(rdi, 80)); + rbx = (word64)(rbx ^ WC_L64(rdi, 88)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 56)); + r14 = (word64)(WC_L64(rdi, -32)); + r15 = (word64)(WC_L64(rdi, 80)); + rbx = (word64)(WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 80) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -8) = (word64)(r15); + /* XOR in constant */ + rcx = (word64)(rcx ^ 0x8a); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -24)); + r13 = (word64)(WC_L64(rdi, 88)); + r14 = (word64)(WC_L64(rdi, 0)); + r15 = (word64)(WC_L64(rdi, -88)); + rbx = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 24) = (word64)(rbp); + WC_S64(rdi, -24) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 8)); + r13 = (word64)(WC_L64(rdi, -80)); + r14 = (word64)(WC_L64(rdi, 32)); + r15 = (word64)(WC_L64(rdi, -56)); + rbx = (word64)(WC_L64(rdi, 96)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 96) = (word64)(rbp); + WC_S64(rdi, 8) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 40)); + r13 = (word64)(WC_L64(rdi, -48)); + r14 = (word64)(WC_L64(rdi, 64)); + r15 = (word64)(WC_L64(rdi, 16)); + rbx = (word64)(WC_L64(rdi, -72)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -72) = (word64)(rbp); + WC_S64(rdi, 40) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 72)); + r10 = (word64)(r10 ^ WC_L64(rdi, -16)); + r11 = (word64)(r11 ^ WC_L64(rdi, -64)); + rax = (word64)(rax ^ WC_L64(rdi, 48)); + r8 = (word64)(r8 ^ WC_L64(rdi, -40)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 72) = (word64)(r12); + WC_S64(rdi, -16) = (word64)(r13); + WC_S64(rdi, -64) = (word64)(r14); + WC_S64(rdi, 48) = (word64)(r15); + WC_S64(rdi, -40) = (word64)(rbx); + /* Round 9 */ + r12 = (word64)(r12 ^ rcx); + r15 = (word64)(r15 ^ WC_L64(rdi, -88)); + r13 = (word64)(r13 ^ WC_L64(rdi, -80)); + rbx = (word64)(rbx ^ WC_L64(rdi, -72)); + r15 = (word64)(r15 ^ WC_L64(rdi, -56)); + r13 = (word64)(r13 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ WC_L64(rdi, -24)); + rbx = (word64)(rbx ^ WC_L64(rdi, -8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ WC_L64(rdi, 8)); + r15 = (word64)(r15 ^ WC_L64(rdi, 16)); + rbx = (word64)(rbx ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 32)); + r12 = (word64)(r12 ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 56)); + r14 = (word64)(r14 ^ WC_L64(rdi, 64)); + r15 = (word64)(r15 ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 88)); + rbx = (word64)(rbx ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 88)); + r14 = (word64)(WC_L64(rdi, 32)); + r15 = (word64)(WC_L64(rdi, 16)); + rbx = (word64)(WC_L64(rdi, -40)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 16) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -40) = (word64)(r15); + /* XOR in constant */ + rcx = (word64)(rcx ^ 0x88); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 80)); + r13 = (word64)(WC_L64(rdi, 24)); + r14 = (word64)(WC_L64(rdi, 8)); + r15 = (word64)(WC_L64(rdi, -48)); + rbx = (word64)(WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -64) = (word64)(rbp); + WC_S64(rdi, 80) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 56)); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, -56)); + r15 = (word64)(WC_L64(rdi, -72)); + rbx = (word64)(WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 72) = (word64)(rbp); + WC_S64(rdi, 56) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -8)); + r13 = (word64)(WC_L64(rdi, -24)); + r14 = (word64)(WC_L64(rdi, -80)); + r15 = (word64)(WC_L64(rdi, 64)); + rbx = (word64)(WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 48) = (word64)(rbp); + WC_S64(rdi, -8) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -32)); + r10 = (word64)(r10 ^ WC_L64(rdi, -88)); + r11 = (word64)(r11 ^ WC_L64(rdi, 96)); + rax = (word64)(rax ^ WC_L64(rdi, 40)); + r8 = (word64)(r8 ^ WC_L64(rdi, -16)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -32) = (word64)(r12); + WC_S64(rdi, -88) = (word64)(r13); + WC_S64(rdi, 96) = (word64)(r14); + WC_S64(rdi, 40) = (word64)(r15); + WC_S64(rdi, -16) = (word64)(rbx); + /* Round 10 */ + r12 = (word64)(r12 ^ rcx); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r15 = (word64)(r15 ^ WC_L64(rdi, -72)); + rbx = (word64)(rbx ^ WC_L64(rdi, -64)); + r14 = (word64)(r14 ^ WC_L64(rdi, -56)); + r15 = (word64)(r15 ^ WC_L64(rdi, -48)); + rbx = (word64)(rbx ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ WC_L64(rdi, -8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 0)); + r14 = (word64)(r14 ^ WC_L64(rdi, 8)); + r15 = (word64)(r15 ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 32)); + rbx = (word64)(rbx ^ WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ WC_L64(rdi, 56)); + r15 = (word64)(r15 ^ WC_L64(rdi, 64)); + rbx = (word64)(rbx ^ WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 88)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 24)); + r14 = (word64)(WC_L64(rdi, -56)); + r15 = (word64)(WC_L64(rdi, 64)); + rbx = (word64)(WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 64) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -16) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x80008009); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 16)); + r13 = (word64)(WC_L64(rdi, -64)); + r14 = (word64)(WC_L64(rdi, 56)); + r15 = (word64)(WC_L64(rdi, -24)); + rbx = (word64)(WC_L64(rdi, 96)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 96) = (word64)(rbp); + WC_S64(rdi, 16) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 88)); + r13 = (word64)(WC_L64(rdi, 8)); + r14 = (word64)(WC_L64(rdi, -72)); + r15 = (word64)(WC_L64(rdi, 48)); + rbx = (word64)(WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -32) = (word64)(rbp); + WC_S64(rdi, 88) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -40)); + r13 = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(WC_L64(rdi, 0)); + r15 = (word64)(WC_L64(rdi, -80)); + rbx = (word64)(WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 40) = (word64)(rbp); + WC_S64(rdi, -40) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 32)); + r10 = (word64)(r10 ^ WC_L64(rdi, -48)); + r11 = (word64)(r11 ^ WC_L64(rdi, 72)); + rax = (word64)(rax ^ WC_L64(rdi, -8)); + r8 = (word64)(r8 ^ WC_L64(rdi, -88)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 32) = (word64)(r12); + WC_S64(rdi, -48) = (word64)(r13); + WC_S64(rdi, 72) = (word64)(r14); + WC_S64(rdi, -8) = (word64)(r15); + WC_S64(rdi, -88) = (word64)(rbx); + /* Round 11 */ + r12 = (word64)(r12 ^ rcx); + r15 = (word64)(r15 ^ WC_L64(rdi, -80)); + r14 = (word64)(r14 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -64)); + r14 = (word64)(r14 ^ WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ WC_L64(rdi, -40)); + rbx = (word64)(rbx ^ WC_L64(rdi, -32)); + r15 = (word64)(r15 ^ WC_L64(rdi, -24)); + rbx = (word64)(rbx ^ WC_L64(rdi, -16)); + r14 = (word64)(r14 ^ WC_L64(rdi, 0)); + r13 = (word64)(r13 ^ WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 24)); + rbx = (word64)(rbx ^ WC_L64(rdi, 40)); + r15 = (word64)(r15 ^ WC_L64(rdi, 48)); + r14 = (word64)(r14 ^ WC_L64(rdi, 56)); + r15 = (word64)(r15 ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ WC_L64(rdi, 88)); + rbx = (word64)(rbx ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -64)); + r14 = (word64)(WC_L64(rdi, -72)); + r15 = (word64)(WC_L64(rdi, -80)); + rbx = (word64)(WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -80) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -88) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000a); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 64)); + r13 = (word64)(WC_L64(rdi, 96)); + r14 = (word64)(WC_L64(rdi, 88)); + r15 = (word64)(WC_L64(rdi, 80)); + rbx = (word64)(WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 72) = (word64)(rbp); + WC_S64(rdi, 64) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 24)); + r13 = (word64)(WC_L64(rdi, 56)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 40)); + rbx = (word64)(WC_L64(rdi, 32)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 32) = (word64)(rbp); + WC_S64(rdi, 24) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -16)); + r13 = (word64)(WC_L64(rdi, 16)); + r14 = (word64)(WC_L64(rdi, 8)); + r15 = (word64)(WC_L64(rdi, 0)); + rbx = (word64)(WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -8) = (word64)(rbp); + WC_S64(rdi, -16) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -56)); + r10 = (word64)(r10 ^ WC_L64(rdi, -24)); + r11 = (word64)(r11 ^ WC_L64(rdi, -32)); + rax = (word64)(rax ^ WC_L64(rdi, -40)); + r8 = (word64)(r8 ^ WC_L64(rdi, -48)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -56) = (word64)(r12); + WC_S64(rdi, -24) = (word64)(r13); + WC_S64(rdi, -32) = (word64)(r14); + WC_S64(rdi, -40) = (word64)(r15); + WC_S64(rdi, -48) = (word64)(rbx); + /* Round 12 */ + r12 = (word64)(r12 ^ rcx); + rbx = (word64)(rbx ^ WC_L64(rdi, -88)); + r15 = (word64)(r15 ^ WC_L64(rdi, -80)); + r14 = (word64)(r14 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ WC_L64(rdi, -16)); + rbx = (word64)(rbx ^ WC_L64(rdi, -8)); + r15 = (word64)(r15 ^ WC_L64(rdi, 0)); + r14 = (word64)(r14 ^ WC_L64(rdi, 8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ WC_L64(rdi, 24)); + rbx = (word64)(rbx ^ WC_L64(rdi, 32)); + r15 = (word64)(r15 ^ WC_L64(rdi, 40)); + r14 = (word64)(r14 ^ WC_L64(rdi, 48)); + r13 = (word64)(r13 ^ WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ WC_L64(rdi, 64)); + rbx = (word64)(rbx ^ WC_L64(rdi, 72)); + r15 = (word64)(r15 ^ WC_L64(rdi, 80)); + r14 = (word64)(r14 ^ WC_L64(rdi, 88)); + r13 = (word64)(r13 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 96)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 0)); + rbx = (word64)(WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 0) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -48) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000808b); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -80)); + r13 = (word64)(WC_L64(rdi, 72)); + r14 = (word64)(WC_L64(rdi, 24)); + r15 = (word64)(WC_L64(rdi, 16)); + rbx = (word64)(WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -32) = (word64)(rbp); + WC_S64(rdi, -80) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -64)); + r13 = (word64)(WC_L64(rdi, 88)); + r14 = (word64)(WC_L64(rdi, 40)); + r15 = (word64)(WC_L64(rdi, -8)); + rbx = (word64)(WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -56) = (word64)(rbp); + WC_S64(rdi, -64) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -88)); + r13 = (word64)(WC_L64(rdi, 64)); + r14 = (word64)(WC_L64(rdi, 56)); + r15 = (word64)(WC_L64(rdi, 8)); + rbx = (word64)(WC_L64(rdi, -40)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -40) = (word64)(rbp); + WC_S64(rdi, -88) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -72)); + r10 = (word64)(r10 ^ WC_L64(rdi, 80)); + r11 = (word64)(r11 ^ WC_L64(rdi, 32)); + rax = (word64)(rax ^ WC_L64(rdi, -16)); + r8 = (word64)(r8 ^ WC_L64(rdi, -24)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -72) = (word64)(r12); + WC_S64(rdi, 80) = (word64)(r13); + WC_S64(rdi, 32) = (word64)(r14); + WC_S64(rdi, -16) = (word64)(r15); + WC_S64(rdi, -24) = (word64)(rbx); + /* Round 13 */ + r12 = (word64)(r12 ^ rcx); + r12 = (word64)(r12 ^ WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ WC_L64(rdi, -64)); + rbx = (word64)(rbx ^ WC_L64(rdi, -56)); + rbx = (word64)(rbx ^ WC_L64(rdi, -48)); + rbx = (word64)(rbx ^ WC_L64(rdi, -40)); + rbx = (word64)(rbx ^ WC_L64(rdi, -32)); + r15 = (word64)(r15 ^ WC_L64(rdi, -8)); + r15 = (word64)(r15 ^ WC_L64(rdi, 0)); + r15 = (word64)(r15 ^ WC_L64(rdi, 8)); + r15 = (word64)(r15 ^ WC_L64(rdi, 16)); + r14 = (word64)(r14 ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 40)); + r14 = (word64)(r14 ^ WC_L64(rdi, 48)); + r14 = (word64)(r14 ^ WC_L64(rdi, 56)); + r13 = (word64)(r13 ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 72)); + r13 = (word64)(r13 ^ WC_L64(rdi, 88)); + r13 = (word64)(r13 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 72)); + r14 = (word64)(WC_L64(rdi, 40)); + r15 = (word64)(WC_L64(rdi, 8)); + rbx = (word64)(WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 8) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -24) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x800000000000008b); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 0)); + r13 = (word64)(WC_L64(rdi, -32)); + r14 = (word64)(WC_L64(rdi, -64)); + r15 = (word64)(WC_L64(rdi, 64)); + rbx = (word64)(WC_L64(rdi, 32)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 32) = (word64)(rbp); + WC_S64(rdi, 0) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 96)); + r13 = (word64)(WC_L64(rdi, 24)); + r14 = (word64)(WC_L64(rdi, -8)); + r15 = (word64)(WC_L64(rdi, -40)); + rbx = (word64)(WC_L64(rdi, -72)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -72) = (word64)(rbp); + WC_S64(rdi, 96) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -48)); + r13 = (word64)(WC_L64(rdi, -80)); + r14 = (word64)(WC_L64(rdi, 88)); + r15 = (word64)(WC_L64(rdi, 56)); + rbx = (word64)(WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -16) = (word64)(rbp); + WC_S64(rdi, -48) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 48)); + r10 = (word64)(r10 ^ WC_L64(rdi, 16)); + r11 = (word64)(r11 ^ WC_L64(rdi, -56)); + rax = (word64)(rax ^ WC_L64(rdi, -88)); + r8 = (word64)(r8 ^ WC_L64(rdi, 80)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 48) = (word64)(r12); + WC_S64(rdi, 16) = (word64)(r13); + WC_S64(rdi, -56) = (word64)(r14); + WC_S64(rdi, -88) = (word64)(r15); + WC_S64(rdi, 80) = (word64)(rbx); + /* Round 14 */ + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ WC_L64(rdi, -80)); + rbx = (word64)(rbx ^ WC_L64(rdi, -72)); + r14 = (word64)(r14 ^ WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ WC_L64(rdi, -48)); + r15 = (word64)(r15 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -32)); + rbx = (word64)(rbx ^ WC_L64(rdi, -24)); + rbx = (word64)(rbx ^ WC_L64(rdi, -16)); + r14 = (word64)(r14 ^ WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 0)); + r15 = (word64)(r15 ^ WC_L64(rdi, 8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 24)); + rbx = (word64)(rbx ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 40)); + r15 = (word64)(r15 ^ WC_L64(rdi, 56)); + r15 = (word64)(r15 ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 88)); + r12 = (word64)(r12 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -32)); + r14 = (word64)(WC_L64(rdi, -8)); + r15 = (word64)(WC_L64(rdi, 56)); + rbx = (word64)(WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 56) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 80) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000000008089); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 8)); + r13 = (word64)(WC_L64(rdi, 32)); + r14 = (word64)(WC_L64(rdi, 96)); + r15 = (word64)(WC_L64(rdi, -80)); + rbx = (word64)(WC_L64(rdi, -56)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -56) = (word64)(rbp); + WC_S64(rdi, 8) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 72)); + r13 = (word64)(WC_L64(rdi, -64)); + r14 = (word64)(WC_L64(rdi, -40)); + r15 = (word64)(WC_L64(rdi, -16)); + rbx = (word64)(WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 48) = (word64)(rbp); + WC_S64(rdi, 72) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -24)); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 24)); + r15 = (word64)(WC_L64(rdi, 88)); + rbx = (word64)(WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -88) = (word64)(rbp); + WC_S64(rdi, -24) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 40)); + r10 = (word64)(r10 ^ WC_L64(rdi, 64)); + r11 = (word64)(r11 ^ WC_L64(rdi, -72)); + rax = (word64)(rax ^ WC_L64(rdi, -48)); + r8 = (word64)(r8 ^ WC_L64(rdi, 16)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 40) = (word64)(r12); + WC_S64(rdi, 64) = (word64)(r13); + WC_S64(rdi, -72) = (word64)(r14); + WC_S64(rdi, -48) = (word64)(r15); + WC_S64(rdi, 16) = (word64)(rbx); + /* Round 15 */ + r12 = (word64)(r12 ^ rcx); + rbx = (word64)(rbx ^ WC_L64(rdi, -88)); + r15 = (word64)(r15 ^ WC_L64(rdi, -80)); + r13 = (word64)(r13 ^ WC_L64(rdi, -64)); + rbx = (word64)(rbx ^ WC_L64(rdi, -56)); + r14 = (word64)(r14 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -32)); + r12 = (word64)(r12 ^ WC_L64(rdi, -24)); + r15 = (word64)(r15 ^ WC_L64(rdi, -16)); + r14 = (word64)(r14 ^ WC_L64(rdi, -8)); + r13 = (word64)(r13 ^ WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ WC_L64(rdi, 8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 24)); + r13 = (word64)(r13 ^ WC_L64(rdi, 32)); + rbx = (word64)(rbx ^ WC_L64(rdi, 48)); + r15 = (word64)(r15 ^ WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ WC_L64(rdi, 72)); + rbx = (word64)(rbx ^ WC_L64(rdi, 80)); + r15 = (word64)(r15 ^ WC_L64(rdi, 88)); + r14 = (word64)(r14 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 32)); + r14 = (word64)(WC_L64(rdi, -40)); + r15 = (word64)(WC_L64(rdi, 88)); + rbx = (word64)(WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 88) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 16) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000000008003); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 56)); + r13 = (word64)(WC_L64(rdi, -56)); + r14 = (word64)(WC_L64(rdi, 72)); + r15 = (word64)(WC_L64(rdi, 0)); + rbx = (word64)(WC_L64(rdi, -72)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -72) = (word64)(rbp); + WC_S64(rdi, 56) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -32)); + r13 = (word64)(WC_L64(rdi, 96)); + r14 = (word64)(WC_L64(rdi, -16)); + r15 = (word64)(WC_L64(rdi, -88)); + rbx = (word64)(WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 40) = (word64)(rbp); + WC_S64(rdi, -32) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 80)); + r13 = (word64)(WC_L64(rdi, 8)); + r14 = (word64)(WC_L64(rdi, -64)); + r15 = (word64)(WC_L64(rdi, 24)); + rbx = (word64)(WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -48) = (word64)(rbp); + WC_S64(rdi, 80) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -8)); + r10 = (word64)(r10 ^ WC_L64(rdi, -80)); + r11 = (word64)(r11 ^ WC_L64(rdi, 48)); + rax = (word64)(rax ^ WC_L64(rdi, -24)); + r8 = (word64)(r8 ^ WC_L64(rdi, 64)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -8) = (word64)(r12); + WC_S64(rdi, -80) = (word64)(r13); + WC_S64(rdi, 48) = (word64)(r14); + WC_S64(rdi, -24) = (word64)(r15); + WC_S64(rdi, 64) = (word64)(rbx); + /* Round 16 */ + r12 = (word64)(r12 ^ rcx); + r15 = (word64)(r15 ^ WC_L64(rdi, -88)); + rbx = (word64)(rbx ^ WC_L64(rdi, -72)); + r14 = (word64)(r14 ^ WC_L64(rdi, -64)); + r13 = (word64)(r13 ^ WC_L64(rdi, -56)); + rbx = (word64)(rbx ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -40)); + r12 = (word64)(r12 ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -16)); + r15 = (word64)(r15 ^ WC_L64(rdi, 0)); + r13 = (word64)(r13 ^ WC_L64(rdi, 8)); + rbx = (word64)(rbx ^ WC_L64(rdi, 16)); + r15 = (word64)(r15 ^ WC_L64(rdi, 24)); + r13 = (word64)(r13 ^ WC_L64(rdi, 32)); + rbx = (word64)(rbx ^ WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ WC_L64(rdi, 56)); + r14 = (word64)(r14 ^ WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ WC_L64(rdi, 80)); + r15 = (word64)(r15 ^ WC_L64(rdi, 88)); + r13 = (word64)(r13 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -56)); + r14 = (word64)(WC_L64(rdi, -16)); + r15 = (word64)(WC_L64(rdi, 24)); + rbx = (word64)(WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 24) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 64) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000000008002); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 88)); + r13 = (word64)(WC_L64(rdi, -72)); + r14 = (word64)(WC_L64(rdi, -32)); + r15 = (word64)(WC_L64(rdi, 8)); + rbx = (word64)(WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 48) = (word64)(rbp); + WC_S64(rdi, 88) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 72)); + r14 = (word64)(WC_L64(rdi, -88)); + r15 = (word64)(WC_L64(rdi, -48)); + rbx = (word64)(WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -8) = (word64)(rbp); + WC_S64(rdi, 32) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 16)); + r13 = (word64)(WC_L64(rdi, 56)); + r14 = (word64)(WC_L64(rdi, 96)); + r15 = (word64)(WC_L64(rdi, -64)); + rbx = (word64)(WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -24) = (word64)(rbp); + WC_S64(rdi, 16) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -40)); + r10 = (word64)(r10 ^ WC_L64(rdi, 0)); + r11 = (word64)(r11 ^ WC_L64(rdi, 40)); + rax = (word64)(rax ^ WC_L64(rdi, 80)); + r8 = (word64)(r8 ^ WC_L64(rdi, -80)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -40) = (word64)(r12); + WC_S64(rdi, 0) = (word64)(r13); + WC_S64(rdi, 40) = (word64)(r14); + WC_S64(rdi, 80) = (word64)(r15); + WC_S64(rdi, -80) = (word64)(rbx); + /* Round 17 */ + r12 = (word64)(r12 ^ rcx); + r14 = (word64)(r14 ^ WC_L64(rdi, -88)); + r13 = (word64)(r13 ^ WC_L64(rdi, -72)); + r15 = (word64)(r15 ^ WC_L64(rdi, -64)); + r13 = (word64)(r13 ^ WC_L64(rdi, -56)); + r15 = (word64)(r15 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -32)); + rbx = (word64)(rbx ^ WC_L64(rdi, -24)); + r14 = (word64)(r14 ^ WC_L64(rdi, -16)); + rbx = (word64)(rbx ^ WC_L64(rdi, -8)); + r15 = (word64)(r15 ^ WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 16)); + r15 = (word64)(r15 ^ WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 32)); + rbx = (word64)(rbx ^ WC_L64(rdi, 48)); + r13 = (word64)(r13 ^ WC_L64(rdi, 56)); + rbx = (word64)(rbx ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 72)); + r12 = (word64)(r12 ^ WC_L64(rdi, 88)); + r14 = (word64)(r14 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -72)); + r14 = (word64)(WC_L64(rdi, -88)); + r15 = (word64)(WC_L64(rdi, -64)); + rbx = (word64)(WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -64) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -80) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000000000080); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 24)); + r13 = (word64)(WC_L64(rdi, 48)); + r14 = (word64)(WC_L64(rdi, 32)); + r15 = (word64)(WC_L64(rdi, 56)); + rbx = (word64)(WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 56) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 40) = (word64)(rbp); + WC_S64(rdi, 24) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -56)); + r13 = (word64)(WC_L64(rdi, -32)); + r14 = (word64)(WC_L64(rdi, -48)); + r15 = (word64)(WC_L64(rdi, -24)); + rbx = (word64)(WC_L64(rdi, -40)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -40) = (word64)(rbp); + WC_S64(rdi, -56) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 64)); + r13 = (word64)(WC_L64(rdi, 88)); + r14 = (word64)(WC_L64(rdi, 72)); + r15 = (word64)(WC_L64(rdi, 96)); + rbx = (word64)(WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 80) = (word64)(rbp); + WC_S64(rdi, 64) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -16)); + r10 = (word64)(r10 ^ WC_L64(rdi, 8)); + r11 = (word64)(r11 ^ WC_L64(rdi, -8)); + rax = (word64)(rax ^ WC_L64(rdi, 16)); + r8 = (word64)(r8 ^ WC_L64(rdi, 0)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -16) = (word64)(r12); + WC_S64(rdi, 8) = (word64)(r13); + WC_S64(rdi, -8) = (word64)(r14); + WC_S64(rdi, 16) = (word64)(r15); + WC_S64(rdi, 0) = (word64)(rbx); + /* Round 18 */ + r12 = (word64)(r12 ^ rcx); + r14 = (word64)(r14 ^ WC_L64(rdi, -88)); + rbx = (word64)(rbx ^ WC_L64(rdi, -80)); + r13 = (word64)(r13 ^ WC_L64(rdi, -72)); + r15 = (word64)(r15 ^ WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ WC_L64(rdi, -56)); + r14 = (word64)(r14 ^ WC_L64(rdi, -48)); + rbx = (word64)(rbx ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -32)); + r15 = (word64)(r15 ^ WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 32)); + rbx = (word64)(rbx ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 48)); + r15 = (word64)(r15 ^ WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ WC_L64(rdi, 64)); + r14 = (word64)(r14 ^ WC_L64(rdi, 72)); + rbx = (word64)(rbx ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 88)); + r15 = (word64)(r15 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 48)); + r14 = (word64)(WC_L64(rdi, -48)); + r15 = (word64)(WC_L64(rdi, 96)); + rbx = (word64)(WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 96) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 0) = (word64)(r15); + /* XOR in constant */ + rcx = (word64)(rcx ^ 0x800a); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -64)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, -56)); + r15 = (word64)(WC_L64(rdi, 88)); + rbx = (word64)(WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 88) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -8) = (word64)(rbp); + WC_S64(rdi, -64) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -72)); + r13 = (word64)(WC_L64(rdi, 32)); + r14 = (word64)(WC_L64(rdi, -24)); + r15 = (word64)(WC_L64(rdi, 80)); + rbx = (word64)(WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -16) = (word64)(rbp); + WC_S64(rdi, -72) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, -80)); + r13 = (word64)(WC_L64(rdi, 24)); + r14 = (word64)(WC_L64(rdi, -32)); + r15 = (word64)(WC_L64(rdi, 72)); + rbx = (word64)(WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 16) = (word64)(rbp); + WC_S64(rdi, -80) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -88)); + r10 = (word64)(r10 ^ WC_L64(rdi, 56)); + r11 = (word64)(r11 ^ WC_L64(rdi, -40)); + rax = (word64)(rax ^ WC_L64(rdi, 64)); + r8 = (word64)(r8 ^ WC_L64(rdi, 8)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -88) = (word64)(r12); + WC_S64(rdi, 56) = (word64)(r13); + WC_S64(rdi, -40) = (word64)(r14); + WC_S64(rdi, 64) = (word64)(r15); + WC_S64(rdi, 8) = (word64)(rbx); + /* Round 19 */ + r12 = (word64)(r12 ^ rcx); + r12 = (word64)(r12 ^ WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ WC_L64(rdi, -72)); + r12 = (word64)(r12 ^ WC_L64(rdi, -64)); + r14 = (word64)(r14 ^ WC_L64(rdi, -56)); + r14 = (word64)(r14 ^ WC_L64(rdi, -48)); + r14 = (word64)(r14 ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -24)); + rbx = (word64)(rbx ^ WC_L64(rdi, -16)); + rbx = (word64)(rbx ^ WC_L64(rdi, -8)); + rbx = (word64)(rbx ^ WC_L64(rdi, 0)); + rbx = (word64)(rbx ^ WC_L64(rdi, 16)); + r13 = (word64)(r13 ^ WC_L64(rdi, 24)); + r13 = (word64)(r13 ^ WC_L64(rdi, 32)); + r13 = (word64)(r13 ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 48)); + r15 = (word64)(r15 ^ WC_L64(rdi, 72)); + r15 = (word64)(r15 ^ WC_L64(rdi, 80)); + r15 = (word64)(r15 ^ WC_L64(rdi, 88)); + r15 = (word64)(r15 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, -24)); + r15 = (word64)(WC_L64(rdi, 72)); + rbx = (word64)(WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -24) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 72) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 8) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x800000008000000a); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 96)); + r13 = (word64)(WC_L64(rdi, -8)); + r14 = (word64)(WC_L64(rdi, -72)); + r15 = (word64)(WC_L64(rdi, 24)); + rbx = (word64)(WC_L64(rdi, -40)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 24) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -40) = (word64)(rbp); + WC_S64(rdi, 96) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 48)); + r13 = (word64)(WC_L64(rdi, -56)); + r14 = (word64)(WC_L64(rdi, 80)); + r15 = (word64)(WC_L64(rdi, 16)); + rbx = (word64)(WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -88) = (word64)(rbp); + WC_S64(rdi, 48) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 0)); + r13 = (word64)(WC_L64(rdi, -64)); + r14 = (word64)(WC_L64(rdi, 32)); + r15 = (word64)(WC_L64(rdi, -32)); + rbx = (word64)(WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 64) = (word64)(rbp); + WC_S64(rdi, 0) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -48)); + r10 = (word64)(r10 ^ WC_L64(rdi, 88)); + r11 = (word64)(r11 ^ WC_L64(rdi, -16)); + rax = (word64)(rax ^ WC_L64(rdi, -80)); + r8 = (word64)(r8 ^ WC_L64(rdi, 56)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -48) = (word64)(r12); + WC_S64(rdi, 88) = (word64)(r13); + WC_S64(rdi, -16) = (word64)(r14); + WC_S64(rdi, -80) = (word64)(r15); + WC_S64(rdi, 56) = (word64)(rbx); + /* Round 20 */ + r12 = (word64)(r12 ^ rcx); + rbx = (word64)(rbx ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -72)); + r13 = (word64)(r13 ^ WC_L64(rdi, -64)); + r13 = (word64)(r13 ^ WC_L64(rdi, -56)); + rbx = (word64)(rbx ^ WC_L64(rdi, -40)); + r15 = (word64)(r15 ^ WC_L64(rdi, -32)); + r14 = (word64)(r14 ^ WC_L64(rdi, -24)); + r13 = (word64)(r13 ^ WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 0)); + rbx = (word64)(rbx ^ WC_L64(rdi, 8)); + r15 = (word64)(r15 ^ WC_L64(rdi, 16)); + r15 = (word64)(r15 ^ WC_L64(rdi, 24)); + r14 = (word64)(r14 ^ WC_L64(rdi, 32)); + r13 = (word64)(r13 ^ WC_L64(rdi, 40)); + r12 = (word64)(r12 ^ WC_L64(rdi, 48)); + rbx = (word64)(rbx ^ WC_L64(rdi, 64)); + r15 = (word64)(r15 ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -8)); + r14 = (word64)(WC_L64(rdi, 80)); + r15 = (word64)(WC_L64(rdi, -32)); + rbx = (word64)(WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 80) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -32) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 56) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000080008081); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 72)); + r13 = (word64)(WC_L64(rdi, -40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, -64)); + rbx = (word64)(WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -64) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -16) = (word64)(rbp); + WC_S64(rdi, 72) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, 40)); + r13 = (word64)(WC_L64(rdi, -72)); + r14 = (word64)(WC_L64(rdi, 16)); + r15 = (word64)(WC_L64(rdi, 64)); + rbx = (word64)(WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -48) = (word64)(rbp); + WC_S64(rdi, 40) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 8)); + r13 = (word64)(WC_L64(rdi, 96)); + r14 = (word64)(WC_L64(rdi, -56)); + r15 = (word64)(WC_L64(rdi, 32)); + rbx = (word64)(WC_L64(rdi, -80)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -80) = (word64)(rbp); + WC_S64(rdi, 8) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, -24)); + r10 = (word64)(r10 ^ WC_L64(rdi, 24)); + r11 = (word64)(r11 ^ WC_L64(rdi, -88)); + rax = (word64)(rax ^ WC_L64(rdi, 0)); + r8 = (word64)(r8 ^ WC_L64(rdi, 88)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, -24) = (word64)(r12); + WC_S64(rdi, 24) = (word64)(r13); + WC_S64(rdi, -88) = (word64)(r14); + WC_S64(rdi, 0) = (word64)(r15); + WC_S64(rdi, 88) = (word64)(rbx); + /* Round 21 */ + r12 = (word64)(r12 ^ rcx); + rbx = (word64)(rbx ^ WC_L64(rdi, -80)); + r13 = (word64)(r13 ^ WC_L64(rdi, -72)); + r15 = (word64)(r15 ^ WC_L64(rdi, -64)); + r14 = (word64)(r14 ^ WC_L64(rdi, -56)); + rbx = (word64)(rbx ^ WC_L64(rdi, -48)); + r13 = (word64)(r13 ^ WC_L64(rdi, -40)); + r15 = (word64)(r15 ^ WC_L64(rdi, -32)); + rbx = (word64)(rbx ^ WC_L64(rdi, -16)); + r13 = (word64)(r13 ^ WC_L64(rdi, -8)); + r12 = (word64)(r12 ^ WC_L64(rdi, 8)); + r14 = (word64)(r14 ^ WC_L64(rdi, 16)); + r15 = (word64)(r15 ^ WC_L64(rdi, 32)); + r12 = (word64)(r12 ^ WC_L64(rdi, 40)); + r14 = (word64)(r14 ^ WC_L64(rdi, 48)); + rbx = (word64)(rbx ^ WC_L64(rdi, 56)); + r15 = (word64)(r15 ^ WC_L64(rdi, 64)); + r12 = (word64)(r12 ^ WC_L64(rdi, 72)); + r14 = (word64)(r14 ^ WC_L64(rdi, 80)); + r13 = (word64)(r13 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -40)); + r14 = (word64)(WC_L64(rdi, 16)); + r15 = (word64)(WC_L64(rdi, 32)); + rbx = (word64)(WC_L64(rdi, 88)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 16) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 32) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 88) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000000008080); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -32)); + r13 = (word64)(WC_L64(rdi, -16)); + r14 = (word64)(WC_L64(rdi, 40)); + r15 = (word64)(WC_L64(rdi, 96)); + rbx = (word64)(WC_L64(rdi, -88)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 96) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -88) = (word64)(rbp); + WC_S64(rdi, -32) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -8)); + r13 = (word64)(WC_L64(rdi, 48)); + r14 = (word64)(WC_L64(rdi, 64)); + r15 = (word64)(WC_L64(rdi, -80)); + rbx = (word64)(WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -24) = (word64)(rbp); + WC_S64(rdi, -8) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 56)); + r13 = (word64)(WC_L64(rdi, 72)); + r14 = (word64)(WC_L64(rdi, -72)); + r15 = (word64)(WC_L64(rdi, -56)); + rbx = (word64)(WC_L64(rdi, 0)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -56) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 0) = (word64)(rbp); + WC_S64(rdi, 56) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 80)); + r10 = (word64)(r10 ^ WC_L64(rdi, -64)); + r11 = (word64)(r11 ^ WC_L64(rdi, -48)); + rax = (word64)(rax ^ WC_L64(rdi, 8)); + r8 = (word64)(r8 ^ WC_L64(rdi, 24)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 80) = (word64)(r12); + WC_S64(rdi, -64) = (word64)(r13); + WC_S64(rdi, -48) = (word64)(r14); + WC_S64(rdi, 8) = (word64)(r15); + WC_S64(rdi, 24) = (word64)(rbx); + /* Round 22 */ + r12 = (word64)(r12 ^ rcx); + rbx = (word64)(rbx ^ WC_L64(rdi, -88)); + r15 = (word64)(r15 ^ WC_L64(rdi, -80)); + r14 = (word64)(r14 ^ WC_L64(rdi, -72)); + r15 = (word64)(r15 ^ WC_L64(rdi, -56)); + r13 = (word64)(r13 ^ WC_L64(rdi, -40)); + r12 = (word64)(r12 ^ WC_L64(rdi, -32)); + rbx = (word64)(rbx ^ WC_L64(rdi, -24)); + r13 = (word64)(r13 ^ WC_L64(rdi, -16)); + r12 = (word64)(r12 ^ WC_L64(rdi, -8)); + rbx = (word64)(rbx ^ WC_L64(rdi, 0)); + r14 = (word64)(r14 ^ WC_L64(rdi, 16)); + r15 = (word64)(r15 ^ WC_L64(rdi, 32)); + r14 = (word64)(r14 ^ WC_L64(rdi, 40)); + r13 = (word64)(r13 ^ WC_L64(rdi, 48)); + r12 = (word64)(r12 ^ WC_L64(rdi, 56)); + r14 = (word64)(r14 ^ WC_L64(rdi, 64)); + r13 = (word64)(r13 ^ WC_L64(rdi, 72)); + rbx = (word64)(rbx ^ WC_L64(rdi, 88)); + r15 = (word64)(r15 ^ WC_L64(rdi, 96)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -16)); + r14 = (word64)(WC_L64(rdi, 64)); + r15 = (word64)(WC_L64(rdi, -56)); + rbx = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -16) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 64) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -56) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, 24) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x80000001); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, -88)); + r14 = (word64)(WC_L64(rdi, -8)); + r15 = (word64)(WC_L64(rdi, 72)); + rbx = (word64)(WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 72) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -48) = (word64)(rbp); + WC_S64(rdi, 32) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -40)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, -80)); + r15 = (word64)(WC_L64(rdi, 0)); + rbx = (word64)(WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 80) = (word64)(rbp); + WC_S64(rdi, -40) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 88)); + r13 = (word64)(WC_L64(rdi, -32)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, -72)); + rbx = (word64)(WC_L64(rdi, 8)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -72) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 8) = (word64)(rbp); + WC_S64(rdi, 88) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 16)); + r10 = (word64)(r10 ^ WC_L64(rdi, 96)); + r11 = (word64)(r11 ^ WC_L64(rdi, -24)); + rax = (word64)(rax ^ WC_L64(rdi, 56)); + r8 = (word64)(r8 ^ WC_L64(rdi, -64)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 16) = (word64)(r12); + WC_S64(rdi, 96) = (word64)(r13); + WC_S64(rdi, -24) = (word64)(r14); + WC_S64(rdi, 56) = (word64)(r15); + WC_S64(rdi, -64) = (word64)(rbx); + /* Round 23 */ + r12 = (word64)(r12 ^ rcx); + r13 = (word64)(r13 ^ WC_L64(rdi, -88)); + r14 = (word64)(r14 ^ WC_L64(rdi, -80)); + r15 = (word64)(r15 ^ WC_L64(rdi, -72)); + r15 = (word64)(r15 ^ WC_L64(rdi, -56)); + rbx = (word64)(rbx ^ WC_L64(rdi, -48)); + r12 = (word64)(r12 ^ WC_L64(rdi, -40)); + r13 = (word64)(r13 ^ WC_L64(rdi, -32)); + r13 = (word64)(r13 ^ WC_L64(rdi, -16)); + r14 = (word64)(r14 ^ WC_L64(rdi, -8)); + r15 = (word64)(r15 ^ WC_L64(rdi, 0)); + rbx = (word64)(rbx ^ WC_L64(rdi, 8)); + rbx = (word64)(rbx ^ WC_L64(rdi, 24)); + r12 = (word64)(r12 ^ WC_L64(rdi, 32)); + r13 = (word64)(r13 ^ WC_L64(rdi, 40)); + r14 = (word64)(r14 ^ WC_L64(rdi, 48)); + r14 = (word64)(r14 ^ WC_L64(rdi, 64)); + r15 = (word64)(r15 ^ WC_L64(rdi, 72)); + rbx = (word64)(rbx ^ WC_L64(rdi, 80)); + r12 = (word64)(r12 ^ WC_L64(rdi, 88)); + /* Calc t[0..4] */ + rax = (word64)(((r13) >> 63) | ((r13) << 1)); + r8 = (word64)(((r14) >> 63) | ((r14) << 1)); + r9 = (word64)(((r15) >> 63) | ((r15) << 1)); + r10 = (word64)(((rbx) >> 63) | ((rbx) << 1)); + r11 = (word64)(((r12) >> 63) | ((r12) << 1)); + rax = (word64)(rax ^ rbx); + r8 = (word64)(r8 ^ r12); + r9 = (word64)(r9 ^ r13); + r10 = (word64)(r10 ^ r14); + r11 = (word64)(r11 ^ r15); + /* Row Mix */ + /* Row 0 */ + r12 = (word64)(rcx); + r13 = (word64)(WC_L64(rdi, -88)); + r14 = (word64)(WC_L64(rdi, -80)); + r15 = (word64)(WC_L64(rdi, -72)); + rbx = (word64)(WC_L64(rdi, -64)); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + r13 = (word64)(((r13) << 44) | ((r13) >> 20)); + r14 = (word64)(((r14) << 43) | ((r14) >> 21)); + r15 = (word64)(((r15) << 21) | ((r15) >> 43)); + rbx = (word64)(((rbx) << 14) | ((rbx) >> 50)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -88) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -80) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -72) = (word64)(rbp); + r15 = (word64)((~r12) & r13); + rcx = (word64)((~r13) & r14); + r15 = (word64)(r15 ^ rbx); + rcx = (word64)(rcx ^ r12); + WC_S64(rdi, -64) = (word64)(r15); + /* XOR in constant */ + rbx = (word64)(0x8000000080008008); + rcx = (word64)(rcx ^ rbx); + /* Row 1 */ + r12 = (word64)(WC_L64(rdi, -56)); + r13 = (word64)(WC_L64(rdi, -48)); + r14 = (word64)(WC_L64(rdi, -40)); + r15 = (word64)(WC_L64(rdi, -32)); + rbx = (word64)(WC_L64(rdi, -24)); + r12 = (word64)(r12 ^ r10); + r13 = (word64)(r13 ^ r11); + r14 = (word64)(r14 ^ rax); + r15 = (word64)(r15 ^ r8); + rbx = (word64)(rbx ^ r9); + r12 = (word64)(((r12) << 28) | ((r12) >> 36)); + r13 = (word64)(((r13) << 20) | ((r13) >> 44)); + r14 = (word64)(((r14) << 3) | ((r14) >> 61)); + r15 = (word64)(((r15) << 45) | ((r15) >> 19)); + rbx = (word64)(((rbx) << 61) | ((rbx) >> 3)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -48) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, -40) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, -32) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, -24) = (word64)(rbp); + WC_S64(rdi, -56) = (word64)(r15); + /* Row 2 */ + r12 = (word64)(WC_L64(rdi, -16)); + r13 = (word64)(WC_L64(rdi, -8)); + r14 = (word64)(WC_L64(rdi, 0)); + r15 = (word64)(WC_L64(rdi, 8)); + rbx = (word64)(WC_L64(rdi, 16)); + r12 = (word64)(r12 ^ r8); + r13 = (word64)(r13 ^ r9); + r14 = (word64)(r14 ^ r10); + r15 = (word64)(r15 ^ r11); + rbx = (word64)(rbx ^ rax); + r12 = (word64)(((r12) << 1) | ((r12) >> 63)); + r13 = (word64)(((r13) << 6) | ((r13) >> 58)); + r14 = (word64)(((r14) << 25) | ((r14) >> 39)); + r15 = (word64)(((r15) << 8) | ((r15) >> 56)); + rbx = (word64)(((rbx) << 18) | ((rbx) >> 46)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, -8) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 0) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 8) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 16) = (word64)(rbp); + WC_S64(rdi, -16) = (word64)(r15); + /* Row 3 */ + r12 = (word64)(WC_L64(rdi, 24)); + r13 = (word64)(WC_L64(rdi, 32)); + r14 = (word64)(WC_L64(rdi, 40)); + r15 = (word64)(WC_L64(rdi, 48)); + rbx = (word64)(WC_L64(rdi, 56)); + r12 = (word64)(r12 ^ r11); + r13 = (word64)(r13 ^ rax); + r14 = (word64)(r14 ^ r8); + r15 = (word64)(r15 ^ r9); + rbx = (word64)(rbx ^ r10); + r12 = (word64)(((r12) << 27) | ((r12) >> 37)); + r13 = (word64)(((r13) << 36) | ((r13) >> 28)); + r14 = (word64)(((r14) << 10) | ((r14) >> 54)); + r15 = (word64)(((r15) << 15) | ((r15) >> 49)); + rbx = (word64)(((rbx) << 56) | ((rbx) >> 8)); + rbp = (word64)((~r14) & r15); + rbp = (word64)(rbp ^ r13); + WC_S64(rdi, 32) = (word64)(rbp); + rbp = (word64)((~r15) & rbx); + rbp = (word64)(rbp ^ r14); + WC_S64(rdi, 40) = (word64)(rbp); + rbp = (word64)((~rbx) & r12); + rbp = (word64)(rbp ^ r15); + WC_S64(rdi, 48) = (word64)(rbp); + rbp = (word64)((~r12) & r13); + r15 = (word64)((~r13) & r14); + rbp = (word64)(rbp ^ rbx); + r15 = (word64)(r15 ^ r12); + WC_S64(rdi, 56) = (word64)(rbp); + WC_S64(rdi, 24) = (word64)(r15); + /* Row 4 */ + r9 = (word64)(r9 ^ WC_L64(rdi, 64)); + r10 = (word64)(r10 ^ WC_L64(rdi, 72)); + r11 = (word64)(r11 ^ WC_L64(rdi, 80)); + rax = (word64)(rax ^ WC_L64(rdi, 88)); + r8 = (word64)(r8 ^ WC_L64(rdi, 96)); + r12 = (word64)(((r9) >> 2) | ((r9) << 62)); + r13 = (word64)(((r10) >> 9) | ((r10) << 55)); + r14 = (word64)(((r11) >> 25) | ((r11) << 39)); + r15 = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(((r8) >> 62) | ((r8) << 2)); + rax = (word64)((~r13) & r14); + r8 = (word64)((~r14) & r15); + r9 = (word64)((~r15) & rbx); + r10 = (word64)((~rbx) & r12); + r11 = (word64)((~r12) & r13); + r12 = (word64)(r12 ^ rax); + r13 = (word64)(r13 ^ r8); + r14 = (word64)(r14 ^ r9); + r15 = (word64)(r15 ^ r10); + rbx = (word64)(rbx ^ r11); + WC_S64(rdi, 64) = (word64)(r12); + WC_S64(rdi, 72) = (word64)(r13); + WC_S64(rdi, 80) = (word64)(r14); + WC_S64(rdi, 88) = (word64)(r15); + WC_S64(rdi, 96) = (word64)(rbx); + rsi = (word64)(rsi + WC_L64(rsp, 0)); + rdx = (word32)((word32)rdx - 1); + zf1 = (word32)rdx; + rbp = (word64)(WC_L64(rsp, 0)); + if ((sword32)(zf1) > (sword32)(0)) { + goto L_sha3_block_n_bmi2_start; + } + WC_S64(rdi, -96) = (word64)(rcx); + rbp = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); +} + +XALIGNED(32) static const word64 L_sha3_block_avx2_rotl[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x000000000000003eULL, + 0x000000000000001cULL, 0x000000000000001bULL, + 0x000000000000002cULL, 0x0000000000000006ULL, + 0x0000000000000037ULL, 0x0000000000000014ULL, + 0x000000000000000aULL, 0x000000000000002bULL, + 0x0000000000000019ULL, 0x0000000000000027ULL, + 0x000000000000002dULL, 0x000000000000000fULL, + 0x0000000000000015ULL, 0x0000000000000008ULL, + 0x0000000000000024ULL, 0x0000000000000003ULL, + 0x0000000000000029ULL, 0x0000000000000012ULL, + 0x0000000000000002ULL, 0x000000000000003dULL, + 0x0000000000000038ULL, 0x000000000000000eULL, +}; + +XALIGNED(32) static const word64 L_sha3_block_avx2_rotr[] WC_X64I_UNUSED = { + 0x000000000000003fULL, 0x0000000000000002ULL, + 0x0000000000000024ULL, 0x0000000000000025ULL, + 0x0000000000000014ULL, 0x000000000000003aULL, + 0x0000000000000009ULL, 0x000000000000002cULL, + 0x0000000000000036ULL, 0x0000000000000015ULL, + 0x0000000000000027ULL, 0x0000000000000019ULL, + 0x0000000000000013ULL, 0x0000000000000031ULL, + 0x000000000000002bULL, 0x0000000000000038ULL, + 0x000000000000001cULL, 0x000000000000003dULL, + 0x0000000000000017ULL, 0x000000000000002eULL, + 0x000000000000003eULL, 0x0000000000000003ULL, + 0x0000000000000008ULL, 0x0000000000000032ULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sha3_block_avx2(word64* s) +{ + word64 rdi, rdx, rax, rcx, r8; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, + y13 = _mm256_setzero_si256(), y14 = _mm256_setzero_si256(), + y15 = _mm256_setzero_si256(); + + rdi = (word64)(size_t)s; + + rdx = (word64)(L_sha3_avx2_r); + rax = (word64)(L_sha3_block_avx2_rotl); + rax = (word64)(rax + 0x40); + rcx = (word64)(L_sha3_block_avx2_rotr); + rcx = (word64)(rcx + 0x40); + r8 = (word64)(0x18); + y0 = _mm256_set1_epi64x((long long)WC_L64(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 8)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 40)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 72)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 104)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 136)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 168)); + y7 = _mm256_permute4x64_epi64(y2, 0x39); + y8 = _mm256_permute4x64_epi64(y3, 0x1e); + y9 = _mm256_permute4x64_epi64(y4, 0x4b); + y10 = _mm256_permute4x64_epi64(y5, 0x93); + y11 = _mm256_blend_epi32(y2, y3, 0xc); + y12 = _mm256_blend_epi32(y4, y5, 0xc0); + y2 = _mm256_blend_epi32(y7, y8, 0xc0); + y3 = _mm256_blend_epi32(y8, y9, 0xf0); + y4 = _mm256_blend_epi32(y10, y9, 3); + y5 = _mm256_blend_epi32(y11, y12, 0xf0); +L_sha3_block_avx2_start: + /* Calc b[0..4] */ + y7 = _mm256_shuffle_epi32(y5, 0xee); + y15 = _mm256_xor_si256(y1, y2); + y14 = _mm256_xor_si256(y5, y7); + y12 = _mm256_xor_si256(y3, y4); + y7 = _mm256_permute4x64_epi64(y14, 0xaa); + y14 = _mm256_xor_si256(y14, y0); + y14 = _mm256_xor_si256(y14, y7); + y15 = _mm256_xor_si256(y15, y6); + y15 = _mm256_xor_si256(y15, y12); + y14 = _mm256_permute4x64_epi64(y14, 0); + /* XOR in b[x+4] */ + y7 = _mm256_permute4x64_epi64(y15, 0x93); + y9 = _mm256_permute4x64_epi64(y15, 0x39); + y10 = _mm256_permute4x64_epi64(y15, 0); + y15 = _mm256_permute4x64_epi64(y15, 0xff); + y9 = _mm256_blend_epi32(y9, y14, 0xc0); + y14 = _mm256_blend_epi32(y7, y14, 3); + /* Rotate left 1 */ + y8 = _mm256_srli_epi64(y10, 63); + y10 = _mm256_add_epi64(y10, y10); + y7 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_add_epi64(y9, y9); + y10 = _mm256_or_si256(y10, y8); + y9 = _mm256_or_si256(y9, y7); + y10 = _mm256_xor_si256(y10, y15); + y9 = _mm256_xor_si256(y9, y14); + /* XOR in ROTL64(b[x+1]) */ + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y9); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y9); + y4 = _mm256_xor_si256(y4, y9); + y5 = _mm256_xor_si256(y5, y10); + y6 = _mm256_xor_si256(y6, y9); + /* Shuffle - Rotate */ + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -64)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, -32)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, -64)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, -32)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 0)); + y7 = _mm256_srlv_epi64(y1, y7); + y9 = _mm256_srlv_epi64(y2, y9); + y11 = _mm256_srlv_epi64(y3, y11); + y1 = _mm256_sllv_epi64(y1, y8); + y2 = _mm256_sllv_epi64(y2, y10); + y3 = _mm256_sllv_epi64(y3, y12); + y1 = _mm256_or_si256(y1, y7); + y2 = _mm256_or_si256(y2, y9); + y3 = _mm256_or_si256(y3, y11); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 32)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 64)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 96)); + y7 = _mm256_srlv_epi64(y4, y7); + y9 = _mm256_srlv_epi64(y5, y9); + y11 = _mm256_srlv_epi64(y6, y11); + y4 = _mm256_sllv_epi64(y4, y8); + y5 = _mm256_sllv_epi64(y5, y10); + y6 = _mm256_sllv_epi64(y6, y12); + y4 = _mm256_or_si256(y4, y7); + y5 = _mm256_or_si256(y5, y9); + y6 = _mm256_or_si256(y6, y11); + /* Row Mix */ + y12 = _mm256_permute4x64_epi64(y2, 0); + y13 = _mm256_permute4x64_epi64(y3, 0x55); + y14 = _mm256_permute4x64_epi64(y4, 0xaa); + y15 = _mm256_permute4x64_epi64(y6, 0xff); + y7 = _mm256_andnot_si256(y13, y14); + y8 = _mm256_andnot_si256(y14, y15); + y9 = _mm256_andnot_si256(y15, y0); + y10 = _mm256_andnot_si256(y0, y12); + y11 = _mm256_andnot_si256(y12, y13); + y12 = _mm256_xor_si256(y12, y7); + y13 = _mm256_xor_si256(y13, y8); + y14 = _mm256_xor_si256(y14, y9); + y15 = _mm256_xor_si256(y15, y10); + y0 = _mm256_xor_si256(y0, y11); + y7 = _mm256_permute4x64_epi64(y5, 0x8d); + y10 = _mm256_blend_epi32(y12, y13, 0xc); + y11 = _mm256_permute4x64_epi64(y1, 0x72); + y9 = _mm256_blend_epi32(y14, y15, 0xc0); + y12 = _mm256_permute4x64_epi64(y2, 0x87); + y1 = _mm256_blend_epi32(y10, y9, 0xf0); + y13 = _mm256_permute4x64_epi64(y3, 0xc9); + y14 = _mm256_permute4x64_epi64(y4, 0x9c); + y15 = _mm256_permute4x64_epi64(y6, 0x2d); + y12 = _mm256_blend_epi32(y12, y7, 0x30); + y13 = _mm256_blend_epi32(y13, y7, 3); + y14 = _mm256_blend_epi32(y14, y7, 0xc0); + y15 = _mm256_blend_epi32(y15, y7, 0xc); + y5 = _mm256_andnot_si256(y12, y13); + y7 = _mm256_andnot_si256(y13, y14); + y2 = _mm256_andnot_si256(y14, y15); + y3 = _mm256_andnot_si256(y15, y11); + y4 = _mm256_andnot_si256(y11, y12); + y5 = _mm256_xor_si256(y11, y5); + y12 = _mm256_xor_si256(y12, y7); + y13 = _mm256_xor_si256(y13, y2); + y14 = _mm256_xor_si256(y14, y3); + y15 = _mm256_xor_si256(y15, y4); + y3 = _mm256_permute2x128_si256(y12, y14, 0x20); + y7 = _mm256_permute2x128_si256(y13, y15, 0x20); + y6 = _mm256_permute2x128_si256(y12, y14, 0x31); + y8 = _mm256_permute2x128_si256(y13, y15, 0x31); + y2 = _mm256_unpacklo_epi64(y3, y7); + y3 = _mm256_unpackhi_epi64(y3, y7); + y4 = _mm256_unpacklo_epi64(y6, y8); + y6 = _mm256_unpackhi_epi64(y6, y8); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + rdx = (word64)(rdx + 0x20); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_sha3_block_avx2_start; + } + y7 = _mm256_permute4x64_epi64(y2, 0x93); + y8 = _mm256_permute4x64_epi64(y3, 0x4e); + y9 = _mm256_permute4x64_epi64(y4, 0x39); + y2 = _mm256_blend_epi32(y7, y5, 3); + y3 = _mm256_blend_epi32(y8, y7, 3); + y3 = _mm256_blend_epi32(y3, y5, 0xc); + y4 = _mm256_blend_epi32(y8, y9, 0xc0); + y4 = _mm256_blend_epi32(y4, y5, 0x30); + y5 = _mm256_blend_epi32(y9, y5, 0xc0); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 8), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 40), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 72), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 104), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 136), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 168), y6); +} + +XALIGNED(32) static const word64 L_sha3_block_n_avx2_rotl[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x000000000000003eULL, + 0x000000000000001cULL, 0x000000000000001bULL, + 0x000000000000002cULL, 0x0000000000000006ULL, + 0x0000000000000037ULL, 0x0000000000000014ULL, + 0x000000000000000aULL, 0x000000000000002bULL, + 0x0000000000000019ULL, 0x0000000000000027ULL, + 0x000000000000002dULL, 0x000000000000000fULL, + 0x0000000000000015ULL, 0x0000000000000008ULL, + 0x0000000000000024ULL, 0x0000000000000003ULL, + 0x0000000000000029ULL, 0x0000000000000012ULL, + 0x0000000000000002ULL, 0x000000000000003dULL, + 0x0000000000000038ULL, 0x000000000000000eULL, +}; + +XALIGNED(32) static const word64 L_sha3_block_n_avx2_rotr[] WC_X64I_UNUSED = { + 0x000000000000003fULL, 0x0000000000000002ULL, + 0x0000000000000024ULL, 0x0000000000000025ULL, + 0x0000000000000014ULL, 0x000000000000003aULL, + 0x0000000000000009ULL, 0x000000000000002cULL, + 0x0000000000000036ULL, 0x0000000000000015ULL, + 0x0000000000000027ULL, 0x0000000000000019ULL, + 0x0000000000000013ULL, 0x0000000000000031ULL, + 0x000000000000002bULL, 0x0000000000000038ULL, + 0x000000000000001cULL, 0x000000000000003dULL, + 0x0000000000000017ULL, 0x000000000000002eULL, + 0x000000000000003eULL, 0x0000000000000003ULL, + 0x0000000000000008ULL, 0x0000000000000032ULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sha3_block_n_avx2(word64* s, const byte* data, word32 n, + word64 c) +{ + word64 rdi, rsi, rdx, rcx, rax, r8, r9, r10; + __m256i y0, y1, y2, y3, y4, y5, y6, y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(); + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)data; + rdx = (word64)(word32)n; + rcx = (word64)(word64)c; + + rax = (word64)(L_sha3_avx2_r); + r8 = (word64)(L_sha3_block_n_avx2_rotl); + r8 = (word64)(r8 + 0x40); + r9 = (word64)(L_sha3_block_n_avx2_rotr); + r9 = (word64)(r9 + 0x40); + y0 = _mm256_set1_epi64x((long long)WC_L64(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 8)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 40)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 72)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 104)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 136)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 168)); + r10 = (word64)(0x18); + if ((rcx) == (0x88)) { + goto L_sha3_block_n_avx2_load_256_1; + } + if ((rcx) == (0xa8)) { + goto L_sha3_block_n_avx2_load_128_1; + } + if ((rcx) == (0x90)) { + goto L_sha3_block_n_avx2_load_224_1; + } + if ((rcx) == (0x68)) { + goto L_sha3_block_n_avx2_load_384_1; + } + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y2 = _mm256_xor_si256(y2, y9); + goto L_sha3_block_n_avx2_start_1; +L_sha3_block_n_avx2_load_128_1: + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 104)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 136)); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y10); + y4 = _mm256_xor_si256(y4, y11); + y5 = _mm256_xor_si256(y5, y12); + goto L_sha3_block_n_avx2_start_1; +L_sha3_block_n_avx2_load_224_1: + y12 = _mm256_setzero_si256(); + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 104)); + y12 = _mm256_zextsi128_si256(_mm_loadl_epi64((const __m128i*)WC_PR(rsi, + 136))); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y10); + y4 = _mm256_xor_si256(y4, y11); + y5 = _mm256_xor_si256(y5, y12); + goto L_sha3_block_n_avx2_start_1; +L_sha3_block_n_avx2_load_384_1: + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y10); + goto L_sha3_block_n_avx2_start_1; +L_sha3_block_n_avx2_load_256_1: + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 104)); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y10); + y4 = _mm256_xor_si256(y4, y11); +L_sha3_block_n_avx2_start_1: + y7 = _mm256_permute4x64_epi64(y2, 0x39); + y8 = _mm256_permute4x64_epi64(y3, 0x1e); + y9 = _mm256_permute4x64_epi64(y4, 0x4b); + y10 = _mm256_permute4x64_epi64(y5, 0x93); + y11 = _mm256_blend_epi32(y2, y3, 0xc); + y12 = _mm256_blend_epi32(y4, y5, 0xc0); + y2 = _mm256_blend_epi32(y7, y8, 0xc0); + y3 = _mm256_blend_epi32(y8, y9, 0xf0); + y4 = _mm256_blend_epi32(y10, y9, 3); + y5 = _mm256_blend_epi32(y11, y12, 0xf0); + goto L_sha3_block_n_avx2_rounds; +L_sha3_block_n_avx2_start: + r10 = (word64)(0x18); + if ((rcx) == (0x88)) { + goto L_sha3_block_n_avx2_load_256; + } + if ((rcx) == (0xa8)) { + goto L_sha3_block_n_avx2_load_128; + } + if ((rcx) == (0x90)) { + goto L_sha3_block_n_avx2_load_224; + } + if ((rcx) == (0x68)) { + goto L_sha3_block_n_avx2_load_384; + } + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y12 = _mm256_setzero_si256(); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y7 = _mm256_permute4x64_epi64(y9, 0x39); + y15 = _mm256_blend_epi32(y9, y12, 0xfc); + y7 = _mm256_blend_epi32(y7, y12, 0xc0); + y2 = _mm256_xor_si256(y2, y7); + y5 = _mm256_xor_si256(y5, y15); + goto L_sha3_block_n_avx2_rounds; +L_sha3_block_n_avx2_load_128: + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 104)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 136)); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y7 = _mm256_permute4x64_epi64(y9, 0x39); + y8 = _mm256_permute4x64_epi64(y10, 0x1e); + y13 = _mm256_permute4x64_epi64(y11, 0x4b); + y14 = _mm256_permute4x64_epi64(y12, 0x93); + y15 = _mm256_blend_epi32(y9, y10, 0xc); + y11 = _mm256_blend_epi32(y11, y12, 0xc0); + y7 = _mm256_blend_epi32(y7, y8, 0xc0); + y8 = _mm256_blend_epi32(y8, y13, 0xf0); + y13 = _mm256_blend_epi32(y14, y13, 3); + y11 = _mm256_blend_epi32(y15, y11, 0xf0); + y2 = _mm256_xor_si256(y2, y7); + y3 = _mm256_xor_si256(y3, y8); + y4 = _mm256_xor_si256(y4, y13); + y5 = _mm256_xor_si256(y5, y11); + goto L_sha3_block_n_avx2_rounds; +L_sha3_block_n_avx2_load_224: + y12 = _mm256_setzero_si256(); + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 104)); + y12 = _mm256_zextsi128_si256(_mm_loadl_epi64((const __m128i*)WC_PR(rsi, + 136))); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y7 = _mm256_permute4x64_epi64(y9, 0x39); + y8 = _mm256_permute4x64_epi64(y10, 0x1e); + y13 = _mm256_permute4x64_epi64(y11, 0x4b); + y14 = _mm256_permute4x64_epi64(y12, 0x93); + y15 = _mm256_blend_epi32(y9, y10, 0xc); + y11 = _mm256_blend_epi32(y11, y12, 0xc0); + y7 = _mm256_blend_epi32(y7, y8, 0xc0); + y8 = _mm256_blend_epi32(y8, y13, 0xf0); + y13 = _mm256_blend_epi32(y14, y13, 3); + y11 = _mm256_blend_epi32(y15, y11, 0xf0); + y2 = _mm256_xor_si256(y2, y7); + y3 = _mm256_xor_si256(y3, y8); + y4 = _mm256_xor_si256(y4, y13); + y5 = _mm256_xor_si256(y5, y11); + goto L_sha3_block_n_avx2_rounds; +L_sha3_block_n_avx2_load_384: + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y12 = _mm256_setzero_si256(); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y7 = _mm256_permute4x64_epi64(y9, 0x39); + y8 = _mm256_permute4x64_epi64(y10, 0x1e); + y13 = _mm256_blend_epi32(y10, y12, 0xf3); + y15 = _mm256_blend_epi32(y9, y13, 0xfc); + y7 = _mm256_blend_epi32(y7, y8, 0xc0); + y8 = _mm256_blend_epi32(y8, y12, 0xf0); + y2 = _mm256_xor_si256(y2, y7); + y3 = _mm256_xor_si256(y3, y8); + y5 = _mm256_xor_si256(y5, y15); + goto L_sha3_block_n_avx2_rounds; +L_sha3_block_n_avx2_load_256: + y7 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 8)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 40)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 104)); + y12 = _mm256_setzero_si256(); + y0 = _mm256_xor_si256(y0, y7); + y1 = _mm256_xor_si256(y1, y8); + y7 = _mm256_permute4x64_epi64(y9, 0x39); + y8 = _mm256_permute4x64_epi64(y10, 0x1e); + y13 = _mm256_permute4x64_epi64(y11, 0x4b); + y15 = _mm256_blend_epi32(y9, y10, 0xc); + y11 = _mm256_blend_epi32(y11, y12, 0xcf); + y7 = _mm256_blend_epi32(y7, y8, 0xc0); + y8 = _mm256_blend_epi32(y8, y13, 0xf0); + y13 = _mm256_blend_epi32(y13, y12, 0xfc); + y11 = _mm256_blend_epi32(y15, y11, 0xf0); + y2 = _mm256_xor_si256(y2, y7); + y3 = _mm256_xor_si256(y3, y8); + y4 = _mm256_xor_si256(y4, y13); + y5 = _mm256_xor_si256(y5, y11); +L_sha3_block_n_avx2_rounds: + /* Calc b[0..4] */ + y7 = _mm256_shuffle_epi32(y5, 0xee); + y15 = _mm256_xor_si256(y1, y2); + y14 = _mm256_xor_si256(y5, y7); + y12 = _mm256_xor_si256(y3, y4); + y7 = _mm256_permute4x64_epi64(y14, 0xaa); + y14 = _mm256_xor_si256(y14, y0); + y14 = _mm256_xor_si256(y14, y7); + y15 = _mm256_xor_si256(y15, y6); + y15 = _mm256_xor_si256(y15, y12); + y14 = _mm256_permute4x64_epi64(y14, 0); + /* XOR in b[x+4] */ + y7 = _mm256_permute4x64_epi64(y15, 0x93); + y9 = _mm256_permute4x64_epi64(y15, 0x39); + y10 = _mm256_permute4x64_epi64(y15, 0); + y15 = _mm256_permute4x64_epi64(y15, 0xff); + y9 = _mm256_blend_epi32(y9, y14, 0xc0); + y14 = _mm256_blend_epi32(y7, y14, 3); + /* Rotate left 1 */ + y8 = _mm256_srli_epi64(y10, 63); + y10 = _mm256_add_epi64(y10, y10); + y7 = _mm256_srli_epi64(y9, 63); + y9 = _mm256_add_epi64(y9, y9); + y10 = _mm256_or_si256(y10, y8); + y9 = _mm256_or_si256(y9, y7); + y10 = _mm256_xor_si256(y10, y15); + y9 = _mm256_xor_si256(y9, y14); + /* XOR in ROTL64(b[x+1]) */ + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y9); + y2 = _mm256_xor_si256(y2, y9); + y3 = _mm256_xor_si256(y3, y9); + y4 = _mm256_xor_si256(y4, y9); + y5 = _mm256_xor_si256(y5, y10); + y6 = _mm256_xor_si256(y6, y9); + /* Shuffle - Rotate */ + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, -64)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, -32)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, -64)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, -32)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 0)); + y7 = _mm256_srlv_epi64(y1, y7); + y9 = _mm256_srlv_epi64(y2, y9); + y11 = _mm256_srlv_epi64(y3, y11); + y1 = _mm256_sllv_epi64(y1, y8); + y2 = _mm256_sllv_epi64(y2, y10); + y3 = _mm256_sllv_epi64(y3, y12); + y1 = _mm256_or_si256(y1, y7); + y2 = _mm256_or_si256(y2, y9); + y3 = _mm256_or_si256(y3, y11); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 32)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 96)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 32)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 64)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 96)); + y7 = _mm256_srlv_epi64(y4, y7); + y9 = _mm256_srlv_epi64(y5, y9); + y11 = _mm256_srlv_epi64(y6, y11); + y4 = _mm256_sllv_epi64(y4, y8); + y5 = _mm256_sllv_epi64(y5, y10); + y6 = _mm256_sllv_epi64(y6, y12); + y4 = _mm256_or_si256(y4, y7); + y5 = _mm256_or_si256(y5, y9); + y6 = _mm256_or_si256(y6, y11); + /* Row Mix */ + y12 = _mm256_permute4x64_epi64(y2, 0); + y13 = _mm256_permute4x64_epi64(y3, 0x55); + y14 = _mm256_permute4x64_epi64(y4, 0xaa); + y15 = _mm256_permute4x64_epi64(y6, 0xff); + y7 = _mm256_andnot_si256(y13, y14); + y8 = _mm256_andnot_si256(y14, y15); + y9 = _mm256_andnot_si256(y15, y0); + y10 = _mm256_andnot_si256(y0, y12); + y11 = _mm256_andnot_si256(y12, y13); + y12 = _mm256_xor_si256(y12, y7); + y13 = _mm256_xor_si256(y13, y8); + y14 = _mm256_xor_si256(y14, y9); + y15 = _mm256_xor_si256(y15, y10); + y0 = _mm256_xor_si256(y0, y11); + y7 = _mm256_permute4x64_epi64(y5, 0x8d); + y10 = _mm256_blend_epi32(y12, y13, 0xc); + y11 = _mm256_permute4x64_epi64(y1, 0x72); + y9 = _mm256_blend_epi32(y14, y15, 0xc0); + y12 = _mm256_permute4x64_epi64(y2, 0x87); + y1 = _mm256_blend_epi32(y10, y9, 0xf0); + y13 = _mm256_permute4x64_epi64(y3, 0xc9); + y14 = _mm256_permute4x64_epi64(y4, 0x9c); + y15 = _mm256_permute4x64_epi64(y6, 0x2d); + y12 = _mm256_blend_epi32(y12, y7, 0x30); + y13 = _mm256_blend_epi32(y13, y7, 3); + y14 = _mm256_blend_epi32(y14, y7, 0xc0); + y15 = _mm256_blend_epi32(y15, y7, 0xc); + y5 = _mm256_andnot_si256(y12, y13); + y7 = _mm256_andnot_si256(y13, y14); + y2 = _mm256_andnot_si256(y14, y15); + y3 = _mm256_andnot_si256(y15, y11); + y4 = _mm256_andnot_si256(y11, y12); + y5 = _mm256_xor_si256(y11, y5); + y12 = _mm256_xor_si256(y12, y7); + y13 = _mm256_xor_si256(y13, y2); + y14 = _mm256_xor_si256(y14, y3); + y15 = _mm256_xor_si256(y15, y4); + y3 = _mm256_permute2x128_si256(y12, y14, 0x20); + y7 = _mm256_permute2x128_si256(y13, y15, 0x20); + y6 = _mm256_permute2x128_si256(y12, y14, 0x31); + y8 = _mm256_permute2x128_si256(y13, y15, 0x31); + y2 = _mm256_unpacklo_epi64(y3, y7); + y3 = _mm256_unpackhi_epi64(y3, y7); + y4 = _mm256_unpacklo_epi64(y6, y8); + y6 = _mm256_unpackhi_epi64(y6, y8); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + rax = (word64)(rax + 0x20); + r10 = (word64)(r10 - 1); + if ((r10) != (0)) { + goto L_sha3_block_n_avx2_rounds; + } + rax = (word64)(rax - 0x300); + rsi = (word64)(rsi + rcx); + rdx = (word32)((word32)rdx - 1); + if (((word32)rdx) != (0)) { + goto L_sha3_block_n_avx2_start; + } + y7 = _mm256_permute4x64_epi64(y2, 0x93); + y8 = _mm256_permute4x64_epi64(y3, 0x4e); + y9 = _mm256_permute4x64_epi64(y4, 0x39); + y2 = _mm256_blend_epi32(y7, y5, 3); + y3 = _mm256_blend_epi32(y8, y7, 3); + y3 = _mm256_blend_epi32(y3, y5, 0xc); + y4 = _mm256_blend_epi32(y8, y9, 0xc0); + y4 = _mm256_blend_epi32(y4, y5, 0x30); + y5 = _mm256_blend_epi32(y9, y5, 0xc0); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 8), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 40), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 72), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 104), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 136), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 168), y6); +} + +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA) || defined(WOLFSSL_HAVE_FRODOKEM) +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sha3_blocksx4_avx2(word64* s) +{ + word64 rdi, rdx, rax, rcx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)s; + + rdx = (word64)(L_sha3_x4_avx2_r); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + rax = (word64)(rdi); + rcx = (word64)(rdi); + rdi = (word64)(rdi + 0x80); + rax = (word64)(rax + 0x180); + rcx = (word64)(rcx + 0x280); + /* Round 0 */ + /* Calc b[0..4] */ + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, -96)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, -64)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, -32)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y10 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Round 1 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Round 2 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Round 3 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Round 4 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Round 5 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Round 6 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Round 7 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Round 8 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Round 9 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Round 10 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Round 11 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Round 12 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Round 13 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Round 14 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Round 15 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Round 16 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 512))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Round 17 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 544))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Round 18 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 576))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Round 19 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 608))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Round 20 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 640))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Round 21 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 672))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Round 22 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 704))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Round 23 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 736))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + rdi = (word64)(rdi - 0x80); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y15); +} + +#endif /* defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA) || defined(WOLFSSL_HAVE_FRODOKEM) */ +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA) +XALIGNED(32) static const word64 L_sha3_128_blockx4_seed_avx2_end_mark[] + WC_X64I_UNUSED = { + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sha3_128_blocksx4_seed_avx2(word64* s, byte* seed) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)seed; + + rdx = (word64)(L_sha3_x4_avx2_r); + rax = (word64)(rdi); + rcx = (word64)(rdi); + y15 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + rdi = (word64)(rdi + 0x80); + y11 = _mm256_set1_epi64x((long long)WC_L64(rsi, 8)); + rax = (word64)(rax + 0x180); + y12 = _mm256_set1_epi64x((long long)WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x280); + y13 = _mm256_set1_epi64x((long long)WC_L64(rsi, 24)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_sha3_128_blockx4_seed_avx2_end_mark, 0)); + y6 = _mm256_setzero_si256(); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y13); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y6); + y10 = _mm256_xor_si256(y15, y5); + /* Round 0 */ + /* Calc b[0..4] */ + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Round 1 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Round 2 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Round 3 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Round 4 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Round 5 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Round 6 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Round 7 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Round 8 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Round 9 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Round 10 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Round 11 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Round 12 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Round 13 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Round 14 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Round 15 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Round 16 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 512))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Round 17 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 544))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Round 18 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 576))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Round 19 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 608))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Round 20 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 640))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Round 21 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 672))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Round 22 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 704))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Round 23 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 736))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + rdi = (word64)(rdi - 0x80); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y15); +} + +#endif /* defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA) */ +#ifdef WOLFSSL_HAVE_FRODOKEM +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sha3_blocksx4_out_avx2(word64* s, byte* out, word32 len) +{ + word64 rdi, rsi, rdx, rax, r12, r9, r10, r11, rcx = 0, r8 = 0, r14 = 0, + r13 = 0, r15 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y14 = _mm256_setzero_si256(), y15 = _mm256_setzero_si256(); + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)out; + rdx = (word64)(word32)len; + + rax = (word64)(L_sha3_x4_avx2_r); + r12 = (word32)((word32)rdx); + r9 = (word64)(rsi); + r9 = (word64)(r9 + r12); + r10 = (word64)(r9); + r10 = (word64)(r10 + r12); + r11 = (word64)(r10); + r11 = (word64)(r11 + r12); + r12 = (word64)(r12 >> 3); +L_sha3_blocksx4_out_avx2_block: + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + rcx = (word64)(rdi); + r8 = (word64)(rdi); + rdi = (word64)(rdi + 0x80); + rcx = (word64)(rcx + 0x180); + r8 = (word64)(r8 + 0x280); + /* Round 0 */ + /* Calc b[0..4] */ + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, -96)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, -64)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, -32)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y10 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y4); + /* Round 1 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Round 2 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Round 3 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Round 4 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Round 5 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 160))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y4); + /* Round 6 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 192))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y4); + /* Round 7 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 224))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Round 8 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 256))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Round 9 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 288))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Round 10 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 320))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Round 11 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 352))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Round 12 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 384))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Round 13 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 416))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y4); + /* Round 14 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 448))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Round 15 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 480))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y4); + /* Round 16 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 512))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Round 17 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 544))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Round 18 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 576))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Round 19 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 608))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y4); + /* Round 20 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 640))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y4); + /* Round 21 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 672))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Round 22 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 704))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Round 23 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 736))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y4); + rdi = (word64)(rdi - 0x80); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y15); + r14 = (word64)(rdi); + r13 = (word64)(0x15); +L_sha3_blocksx4_out_avx2_word: + r15 = (word64)(WC_L64(r14, 0)); + WC_S64(rsi, 0) = (word64)(r15); + r15 = (word64)(WC_L64(r14, 8)); + WC_S64(r9, 0) = (word64)(r15); + r15 = (word64)(WC_L64(r14, 16)); + WC_S64(r10, 0) = (word64)(r15); + r15 = (word64)(WC_L64(r14, 24)); + WC_S64(r11, 0) = (word64)(r15); + r14 = (word64)(r14 + 0x20); + rsi = (word64)(rsi + 8); + r9 = (word64)(r9 + 8); + r10 = (word64)(r10 + 8); + r11 = (word64)(r11 + 8); + r12 = (word64)(r12 - 1); + if ((r12) == (0)) { + goto L_sha3_blocksx4_out_avx2_done; + } + r13 = (word64)(r13 - 1); + if ((r13) != (0)) { + goto L_sha3_blocksx4_out_avx2_word; + } + goto L_sha3_blocksx4_out_avx2_block; +L_sha3_blocksx4_out_avx2_done: + ; +} + +#endif /* WOLFSSL_HAVE_FRODOKEM */ +#ifdef WOLFSSL_HAVE_MLKEM +XALIGNED(32) static const word64 L_sha3_256_blockx4_seed_avx2_end_mark[] + WC_X64I_UNUSED = { + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sha3_256_blocksx4_seed_avx2(word64* s, byte* seed) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)seed; + + rdx = (word64)(L_sha3_x4_avx2_r); + rax = (word64)(rdi); + rcx = (word64)(rdi); + y15 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + rdi = (word64)(rdi + 0x80); + y11 = _mm256_set1_epi64x((long long)WC_L64(rsi, 8)); + rax = (word64)(rax + 0x180); + y12 = _mm256_set1_epi64x((long long)WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x280); + y13 = _mm256_set1_epi64x((long long)WC_L64(rsi, 24)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_sha3_256_blockx4_seed_avx2_end_mark, 0)); + y6 = _mm256_setzero_si256(); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y13); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y6); + y10 = y15; + y11 = _mm256_xor_si256(y11, y5); + /* Round 0 */ + /* Calc b[0..4] */ + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Round 1 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Round 2 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Round 3 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Round 4 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Round 5 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Round 6 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Round 7 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Round 8 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Round 9 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Round 10 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Round 11 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Round 12 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Round 13 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Round 14 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Round 15 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Round 16 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 512))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Round 17 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 544))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Round 18 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 576))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Round 19 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 608))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Round 20 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 640))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Round 21 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 672))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Round 22 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 704))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Round 23 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 736))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + rdi = (word64)(rdi - 0x80); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y15); +} + +#endif /* WOLFSSL_HAVE_MLKEM */ +#ifdef WOLFSSL_HAVE_MLDSA +XALIGNED(32) static const word64 L_sha3_256_blockx4_seed_64_avx2_end_mark[] + WC_X64I_UNUSED = { + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sha3_256_blocksx4_seed_64_avx2(word64* s, byte* seed) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)seed; + + rdx = (word64)(L_sha3_x4_avx2_r); + rax = (word64)(rdi); + rcx = (word64)(rdi); + y15 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + rdi = (word64)(rdi + 0x80); + y11 = _mm256_set1_epi64x((long long)WC_L64(rsi, 8)); + rax = (word64)(rax + 0x180); + y12 = _mm256_set1_epi64x((long long)WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x280); + y13 = _mm256_set1_epi64x((long long)WC_L64(rsi, 24)); + y14 = _mm256_set1_epi64x((long long)WC_L64(rsi, 32)); + y0 = _mm256_set1_epi64x((long long)WC_L64(rsi, 40)); + y1 = _mm256_set1_epi64x((long long)WC_L64(rsi, 48)); + y2 = _mm256_set1_epi64x((long long)WC_L64(rsi, 56)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y13); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + y4 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_sha3_256_blockx4_seed_64_avx2_end_mark, 0)); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y6); + y10 = y15; + /* Round 0 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y15, y0); + y11 = _mm256_xor_si256(y11, y1); + y12 = _mm256_xor_si256(y12, y2); + y13 = _mm256_xor_si256(y13, y3); + y14 = _mm256_xor_si256(y14, y4); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Round 1 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Round 2 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Round 3 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Round 4 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Round 5 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Round 6 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Round 7 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Round 8 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Round 9 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Round 10 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Round 11 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Round 12 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Round 13 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Round 14 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Round 15 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Round 16 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 512))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Round 17 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 544))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Round 18 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 576))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Round 19 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 608))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Round 20 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 640))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Round 21 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 672))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Round 22 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 704))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Round 23 */ + /* Calc b[0..4] */ + y10 = _mm256_xor_si256(y0, y15); + y11 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y14 = _mm256_xor_si256(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y11 = _mm256_xor_si256(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y12 = _mm256_xor_si256(y12, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y13 = _mm256_xor_si256(y13, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y14 = _mm256_xor_si256(y14, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y10 = _mm256_xor_si256(y10, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + /* Calc t[0..4] */ + y0 = _mm256_srli_epi64(y11, 63); + y1 = _mm256_srli_epi64(y12, 63); + y2 = _mm256_srli_epi64(y13, 63); + y3 = _mm256_srli_epi64(y14, 63); + y4 = _mm256_srli_epi64(y10, 63); + y5 = _mm256_add_epi64(y11, y11); + y6 = _mm256_add_epi64(y12, y12); + y7 = _mm256_add_epi64(y13, y13); + y8 = _mm256_add_epi64(y14, y14); + y9 = _mm256_add_epi64(y10, y10); + y5 = _mm256_or_si256(y5, y0); + y6 = _mm256_or_si256(y6, y1); + y7 = _mm256_or_si256(y7, y2); + y8 = _mm256_or_si256(y8, y3); + y9 = _mm256_or_si256(y9, y4); + y5 = _mm256_xor_si256(y5, y14); + y6 = _mm256_xor_si256(y6, y10); + y7 = _mm256_xor_si256(y7, y11); + y8 = _mm256_xor_si256(y8, y12); + y9 = _mm256_xor_si256(y9, y13); + /* Row Mix */ + /* Row 0 */ + y10 = _mm256_xor_si256(y5, y15); + y11 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -96))); + y12 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -64))); + y13 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + -32))); + y14 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + y0 = _mm256_srli_epi64(y11, 20); + y1 = _mm256_srli_epi64(y12, 21); + y2 = _mm256_srli_epi64(y13, 43); + y3 = _mm256_srli_epi64(y14, 50); + y11 = _mm256_slli_epi64(y11, 44); + y12 = _mm256_slli_epi64(y12, 43); + y13 = _mm256_slli_epi64(y13, 21); + y14 = _mm256_slli_epi64(y14, 14); + y11 = _mm256_or_si256(y11, y0); + y12 = _mm256_or_si256(y12, y1); + y13 = _mm256_or_si256(y13, y2); + y14 = _mm256_or_si256(y14, y3); + y15 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y15 = _mm256_xor_si256(y15, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + /* XOR in constant */ + y15 = _mm256_xor_si256(y15, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 736))); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, -32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + /* Row 1 */ + y10 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + y11 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + y12 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + y13 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + y14 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -96))); + y0 = _mm256_srli_epi64(y10, 36); + y1 = _mm256_srli_epi64(y11, 44); + y2 = _mm256_srli_epi64(y12, 61); + y3 = _mm256_srli_epi64(y13, 19); + y4 = _mm256_srli_epi64(y14, 3); + y10 = _mm256_slli_epi64(y10, 28); + y11 = _mm256_slli_epi64(y11, 20); + y12 = _mm256_slli_epi64(y12, 3); + y13 = _mm256_slli_epi64(y13, 45); + y14 = _mm256_slli_epi64(y14, 61); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -96), y4); + /* Row 2 */ + y10 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -64))); + y11 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + -32))); + y12 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + y13 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + y14 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64))); + y0 = _mm256_srli_epi64(y10, 63); + y1 = _mm256_srli_epi64(y11, 58); + y2 = _mm256_srli_epi64(y12, 39); + y3 = _mm256_srli_epi64(y13, 56); + y4 = _mm256_srli_epi64(y14, 46); + y10 = _mm256_add_epi64(y10, y10); + y11 = _mm256_slli_epi64(y11, 6); + y12 = _mm256_slli_epi64(y12, 25); + y13 = _mm256_slli_epi64(y13, 8); + y14 = _mm256_slli_epi64(y14, 18); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, -32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 64), y4); + /* Row 3 */ + y10 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96))); + y11 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + y12 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -96))); + y13 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -64))); + y14 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + -32))); + y0 = _mm256_srli_epi64(y10, 37); + y1 = _mm256_srli_epi64(y11, 28); + y2 = _mm256_srli_epi64(y12, 54); + y3 = _mm256_srli_epi64(y13, 49); + y4 = _mm256_srli_epi64(y14, 8); + y10 = _mm256_slli_epi64(y10, 27); + y11 = _mm256_slli_epi64(y11, 36); + y12 = _mm256_slli_epi64(y12, 10); + y13 = _mm256_slli_epi64(y13, 15); + y14 = _mm256_slli_epi64(y14, 56); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rax, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, -32), y4); + /* Row 4 */ + y10 = _mm256_xor_si256(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + y11 = _mm256_xor_si256(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + y12 = _mm256_xor_si256(y9, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64))); + y13 = _mm256_xor_si256(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96))); + y14 = _mm256_xor_si256(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + y0 = _mm256_srli_epi64(y10, 2); + y1 = _mm256_srli_epi64(y11, 9); + y2 = _mm256_srli_epi64(y12, 25); + y3 = _mm256_srli_epi64(y13, 23); + y4 = _mm256_srli_epi64(y14, 62); + y10 = _mm256_slli_epi64(y10, 62); + y11 = _mm256_slli_epi64(y11, 55); + y12 = _mm256_slli_epi64(y12, 39); + y13 = _mm256_slli_epi64(y13, 41); + y14 = _mm256_slli_epi64(y14, 2); + y10 = _mm256_or_si256(y10, y0); + y11 = _mm256_or_si256(y11, y1); + y12 = _mm256_or_si256(y12, y2); + y13 = _mm256_or_si256(y13, y3); + y14 = _mm256_or_si256(y14, y4); + y0 = _mm256_andnot_si256(y11, y12); + y1 = _mm256_andnot_si256(y12, y13); + y2 = _mm256_andnot_si256(y13, y14); + y3 = _mm256_andnot_si256(y14, y10); + y4 = _mm256_andnot_si256(y10, y11); + y0 = _mm256_xor_si256(y0, y10); + y1 = _mm256_xor_si256(y1, y11); + y2 = _mm256_xor_si256(y2, y12); + y3 = _mm256_xor_si256(y3, y13); + y4 = _mm256_xor_si256(y4, y14); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y4); + rdi = (word64)(rdi - 0x80); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y15); +} + +#endif /* WOLFSSL_HAVE_MLDSA */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifdef HAVE_INTEL_AVX512 +#if defined(WOLFSSL_HAVE_FRODOKEM) || defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) +XALIGNED(32) static const word64 L_sha3_x8_avx512_r[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000001ULL, + 0x0000000000000001ULL, 0x0000000000000001ULL, + 0x0000000000000001ULL, 0x0000000000000001ULL, + 0x0000000000000001ULL, 0x0000000000000001ULL, + 0x0000000000008082ULL, 0x0000000000008082ULL, + 0x0000000000008082ULL, 0x0000000000008082ULL, + 0x0000000000008082ULL, 0x0000000000008082ULL, + 0x0000000000008082ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x800000000000808aULL, + 0x800000000000808aULL, 0x800000000000808aULL, + 0x800000000000808aULL, 0x800000000000808aULL, + 0x800000000000808aULL, 0x800000000000808aULL, + 0x8000000080008000ULL, 0x8000000080008000ULL, + 0x8000000080008000ULL, 0x8000000080008000ULL, + 0x8000000080008000ULL, 0x8000000080008000ULL, + 0x8000000080008000ULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x000000000000808bULL, + 0x000000000000808bULL, 0x000000000000808bULL, + 0x000000000000808bULL, 0x000000000000808bULL, + 0x000000000000808bULL, 0x000000000000808bULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000000008009ULL, 0x8000000000008009ULL, + 0x8000000000008009ULL, 0x8000000000008009ULL, + 0x8000000000008009ULL, 0x8000000000008009ULL, + 0x8000000000008009ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x000000000000008aULL, + 0x000000000000008aULL, 0x000000000000008aULL, + 0x000000000000008aULL, 0x000000000000008aULL, + 0x000000000000008aULL, 0x000000000000008aULL, + 0x0000000000000088ULL, 0x0000000000000088ULL, + 0x0000000000000088ULL, 0x0000000000000088ULL, + 0x0000000000000088ULL, 0x0000000000000088ULL, + 0x0000000000000088ULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x0000000080008009ULL, + 0x0000000080008009ULL, 0x0000000080008009ULL, + 0x0000000080008009ULL, 0x0000000080008009ULL, + 0x0000000080008009ULL, 0x0000000080008009ULL, + 0x000000008000000aULL, 0x000000008000000aULL, + 0x000000008000000aULL, 0x000000008000000aULL, + 0x000000008000000aULL, 0x000000008000000aULL, + 0x000000008000000aULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x000000008000808bULL, + 0x000000008000808bULL, 0x000000008000808bULL, + 0x000000008000808bULL, 0x000000008000808bULL, + 0x000000008000808bULL, 0x000000008000808bULL, + 0x800000000000008bULL, 0x800000000000008bULL, + 0x800000000000008bULL, 0x800000000000008bULL, + 0x800000000000008bULL, 0x800000000000008bULL, + 0x800000000000008bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008089ULL, + 0x8000000000008089ULL, 0x8000000000008089ULL, + 0x8000000000008089ULL, 0x8000000000008089ULL, + 0x8000000000008089ULL, 0x8000000000008089ULL, + 0x8000000000008003ULL, 0x8000000000008003ULL, + 0x8000000000008003ULL, 0x8000000000008003ULL, + 0x8000000000008003ULL, 0x8000000000008003ULL, + 0x8000000000008003ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000008002ULL, + 0x8000000000008002ULL, 0x8000000000008002ULL, + 0x8000000000008002ULL, 0x8000000000008002ULL, + 0x8000000000008002ULL, 0x8000000000008002ULL, + 0x8000000000000080ULL, 0x8000000000000080ULL, + 0x8000000000000080ULL, 0x8000000000000080ULL, + 0x8000000000000080ULL, 0x8000000000000080ULL, + 0x8000000000000080ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x000000000000800aULL, + 0x000000000000800aULL, 0x000000000000800aULL, + 0x000000000000800aULL, 0x000000000000800aULL, + 0x000000000000800aULL, 0x000000000000800aULL, + 0x800000008000000aULL, 0x800000008000000aULL, + 0x800000008000000aULL, 0x800000008000000aULL, + 0x800000008000000aULL, 0x800000008000000aULL, + 0x800000008000000aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000080008081ULL, 0x8000000080008081ULL, + 0x8000000000008080ULL, 0x8000000000008080ULL, + 0x8000000000008080ULL, 0x8000000000008080ULL, + 0x8000000000008080ULL, 0x8000000000008080ULL, + 0x8000000000008080ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x0000000080000001ULL, 0x0000000080000001ULL, + 0x8000000080008008ULL, 0x8000000080008008ULL, + 0x8000000080008008ULL, 0x8000000080008008ULL, + 0x8000000080008008ULL, 0x8000000080008008ULL, + 0x8000000080008008ULL, 0x8000000080008008ULL, +}; + +#endif /* defined(WOLFSSL_HAVE_FRODOKEM) || defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) */ +#ifdef WOLFSSL_HAVE_FRODOKEM +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void sha3_blocksx8_out_avx512(word64* s, byte* out, word32 len) +{ + word64 rdi, rsi, rdx, rax, r14, rcx, r8, r9, r10, r11, r12, r13, rbx = 0, + r15 = 0, rbp = 0; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23, z24, + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(), + z27 = _mm512_setzero_si512(), z28 = _mm512_setzero_si512(), + z29 = _mm512_setzero_si512(), z30 = _mm512_setzero_si512(), + z31 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)out; + rdx = (word64)(word32)len; + + rax = (word64)(L_sha3_x8_avx512_r); + r14 = (word32)((word32)rdx); + rcx = (word64)(rsi); + rcx = (word64)(rcx + r14); + r8 = (word64)(rcx); + r8 = (word64)(r8 + r14); + r9 = (word64)(r8); + r9 = (word64)(r9 + r14); + r10 = (word64)(r9); + r10 = (word64)(r10 + r14); + r11 = (word64)(r10); + r11 = (word64)(r11 + r14); + r12 = (word64)(r11); + r12 = (word64)(r12 + r14); + r13 = (word64)(r12); + r13 = (word64)(r13 + r14); + r14 = (word64)(r14 >> 3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z16 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1024)); + z17 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1088)); + z18 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1152)); + z19 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1216)); + z20 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1280)); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1344)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1408)); + z23 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1472)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1536)); +L_sha3_blocksx8_out_avx512_block: + /* Round 0 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z5, z10, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z15, z20, 0x96); + z26 = z1; + z26 = _mm512_ternarylogic_epi64(z26, z6, z11, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z16, z21, 0x96); + z27 = z2; + z27 = _mm512_ternarylogic_epi64(z27, z7, z12, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z17, z22, 0x96); + z28 = z3; + z28 = _mm512_ternarylogic_epi64(z28, z8, z13, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z18, z23, 0x96); + z29 = z4; + z29 = _mm512_ternarylogic_epi64(z29, z9, z14, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z19, z24, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z5 = _mm512_xor_si512(z5, z30); + z10 = _mm512_xor_si512(z10, z30); + z15 = _mm512_xor_si512(z15, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z1 = _mm512_xor_si512(z1, z30); + z6 = _mm512_xor_si512(z6, z30); + z11 = _mm512_xor_si512(z11, z30); + z16 = _mm512_xor_si512(z16, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z2 = _mm512_xor_si512(z2, z30); + z7 = _mm512_xor_si512(z7, z30); + z12 = _mm512_xor_si512(z12, z30); + z17 = _mm512_xor_si512(z17, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z3 = _mm512_xor_si512(z3, z30); + z8 = _mm512_xor_si512(z8, z30); + z13 = _mm512_xor_si512(z13, z30); + z18 = _mm512_xor_si512(z18, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z4 = _mm512_xor_si512(z4, z30); + z9 = _mm512_xor_si512(z9, z30); + z14 = _mm512_xor_si512(z14, z30); + z19 = _mm512_xor_si512(z19, z30); + z24 = _mm512_xor_si512(z24, z30); + /* Rho - rotate each lane in place */ + z1 = _mm512_rol_epi64(z1, 1); + z2 = _mm512_rol_epi64(z2, 62); + z3 = _mm512_rol_epi64(z3, 28); + z4 = _mm512_rol_epi64(z4, 27); + z5 = _mm512_rol_epi64(z5, 36); + z6 = _mm512_rol_epi64(z6, 44); + z7 = _mm512_rol_epi64(z7, 6); + z8 = _mm512_rol_epi64(z8, 55); + z9 = _mm512_rol_epi64(z9, 20); + z10 = _mm512_rol_epi64(z10, 3); + z11 = _mm512_rol_epi64(z11, 10); + z12 = _mm512_rol_epi64(z12, 43); + z13 = _mm512_rol_epi64(z13, 25); + z14 = _mm512_rol_epi64(z14, 39); + z15 = _mm512_rol_epi64(z15, 41); + z16 = _mm512_rol_epi64(z16, 45); + z17 = _mm512_rol_epi64(z17, 15); + z18 = _mm512_rol_epi64(z18, 21); + z19 = _mm512_rol_epi64(z19, 8); + z20 = _mm512_rol_epi64(z20, 18); + z21 = _mm512_rol_epi64(z21, 2); + z22 = _mm512_rol_epi64(z22, 61); + z23 = _mm512_rol_epi64(z23, 56); + z24 = _mm512_rol_epi64(z24, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z6; + z0 = _mm512_ternarylogic_epi64(z0, z6, z12, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z12, z18, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z18, z24, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z3; + z31 = z9; + z3 = _mm512_ternarylogic_epi64(z3, z9, z10, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z10, z16, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z16, z22, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z1; + z31 = z7; + z1 = _mm512_ternarylogic_epi64(z1, z7, z13, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z13, z19, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z19, z20, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z4; + z31 = z5; + z4 = _mm512_ternarylogic_epi64(z4, z5, z11, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z11, z17, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z17, z23, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z2; + z31 = z8; + z2 = _mm512_ternarylogic_epi64(z2, z8, z14, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z14, z15, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z15, z21, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 0))); + /* Round 1 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z3, z1, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z4, z2, 0x96); + z26 = z6; + z26 = _mm512_ternarylogic_epi64(z26, z9, z7, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z5, z8, 0x96); + z27 = z12; + z27 = _mm512_ternarylogic_epi64(z27, z10, z13, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z11, z14, 0x96); + z28 = z18; + z28 = _mm512_ternarylogic_epi64(z28, z16, z19, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z17, z15, 0x96); + z29 = z24; + z29 = _mm512_ternarylogic_epi64(z29, z22, z20, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z23, z21, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z3 = _mm512_xor_si512(z3, z30); + z1 = _mm512_xor_si512(z1, z30); + z4 = _mm512_xor_si512(z4, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z6 = _mm512_xor_si512(z6, z30); + z9 = _mm512_xor_si512(z9, z30); + z7 = _mm512_xor_si512(z7, z30); + z5 = _mm512_xor_si512(z5, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z12 = _mm512_xor_si512(z12, z30); + z10 = _mm512_xor_si512(z10, z30); + z13 = _mm512_xor_si512(z13, z30); + z11 = _mm512_xor_si512(z11, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z18 = _mm512_xor_si512(z18, z30); + z16 = _mm512_xor_si512(z16, z30); + z19 = _mm512_xor_si512(z19, z30); + z17 = _mm512_xor_si512(z17, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z24 = _mm512_xor_si512(z24, z30); + z22 = _mm512_xor_si512(z22, z30); + z20 = _mm512_xor_si512(z20, z30); + z23 = _mm512_xor_si512(z23, z30); + z21 = _mm512_xor_si512(z21, z30); + /* Rho - rotate each lane in place */ + z6 = _mm512_rol_epi64(z6, 1); + z12 = _mm512_rol_epi64(z12, 62); + z18 = _mm512_rol_epi64(z18, 28); + z24 = _mm512_rol_epi64(z24, 27); + z3 = _mm512_rol_epi64(z3, 36); + z9 = _mm512_rol_epi64(z9, 44); + z10 = _mm512_rol_epi64(z10, 6); + z16 = _mm512_rol_epi64(z16, 55); + z22 = _mm512_rol_epi64(z22, 20); + z1 = _mm512_rol_epi64(z1, 3); + z7 = _mm512_rol_epi64(z7, 10); + z13 = _mm512_rol_epi64(z13, 43); + z19 = _mm512_rol_epi64(z19, 25); + z20 = _mm512_rol_epi64(z20, 39); + z4 = _mm512_rol_epi64(z4, 41); + z5 = _mm512_rol_epi64(z5, 45); + z11 = _mm512_rol_epi64(z11, 15); + z17 = _mm512_rol_epi64(z17, 21); + z23 = _mm512_rol_epi64(z23, 8); + z2 = _mm512_rol_epi64(z2, 18); + z8 = _mm512_rol_epi64(z8, 2); + z14 = _mm512_rol_epi64(z14, 61); + z15 = _mm512_rol_epi64(z15, 56); + z21 = _mm512_rol_epi64(z21, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z9; + z0 = _mm512_ternarylogic_epi64(z0, z9, z13, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z13, z17, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z17, z21, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z18; + z31 = z22; + z18 = _mm512_ternarylogic_epi64(z18, z22, z1, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z1, z5, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z5, z14, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z6; + z31 = z10; + z6 = _mm512_ternarylogic_epi64(z6, z10, z19, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z19, z23, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z23, z2, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z24; + z31 = z3; + z24 = _mm512_ternarylogic_epi64(z24, z3, z7, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z7, z11, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z11, z15, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z12; + z31 = z16; + z12 = _mm512_ternarylogic_epi64(z12, z16, z20, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z20, z4, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z4, z8, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 64))); + /* Round 2 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z18, z6, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z24, z12, 0x96); + z26 = z9; + z26 = _mm512_ternarylogic_epi64(z26, z22, z10, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z3, z16, 0x96); + z27 = z13; + z27 = _mm512_ternarylogic_epi64(z27, z1, z19, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z7, z20, 0x96); + z28 = z17; + z28 = _mm512_ternarylogic_epi64(z28, z5, z23, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z11, z4, 0x96); + z29 = z21; + z29 = _mm512_ternarylogic_epi64(z29, z14, z2, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z15, z8, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z18 = _mm512_xor_si512(z18, z30); + z6 = _mm512_xor_si512(z6, z30); + z24 = _mm512_xor_si512(z24, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z9 = _mm512_xor_si512(z9, z30); + z22 = _mm512_xor_si512(z22, z30); + z10 = _mm512_xor_si512(z10, z30); + z3 = _mm512_xor_si512(z3, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z13 = _mm512_xor_si512(z13, z30); + z1 = _mm512_xor_si512(z1, z30); + z19 = _mm512_xor_si512(z19, z30); + z7 = _mm512_xor_si512(z7, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z17 = _mm512_xor_si512(z17, z30); + z5 = _mm512_xor_si512(z5, z30); + z23 = _mm512_xor_si512(z23, z30); + z11 = _mm512_xor_si512(z11, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z21 = _mm512_xor_si512(z21, z30); + z14 = _mm512_xor_si512(z14, z30); + z2 = _mm512_xor_si512(z2, z30); + z15 = _mm512_xor_si512(z15, z30); + z8 = _mm512_xor_si512(z8, z30); + /* Rho - rotate each lane in place */ + z9 = _mm512_rol_epi64(z9, 1); + z13 = _mm512_rol_epi64(z13, 62); + z17 = _mm512_rol_epi64(z17, 28); + z21 = _mm512_rol_epi64(z21, 27); + z18 = _mm512_rol_epi64(z18, 36); + z22 = _mm512_rol_epi64(z22, 44); + z1 = _mm512_rol_epi64(z1, 6); + z5 = _mm512_rol_epi64(z5, 55); + z14 = _mm512_rol_epi64(z14, 20); + z6 = _mm512_rol_epi64(z6, 3); + z10 = _mm512_rol_epi64(z10, 10); + z19 = _mm512_rol_epi64(z19, 43); + z23 = _mm512_rol_epi64(z23, 25); + z2 = _mm512_rol_epi64(z2, 39); + z24 = _mm512_rol_epi64(z24, 41); + z3 = _mm512_rol_epi64(z3, 45); + z7 = _mm512_rol_epi64(z7, 15); + z11 = _mm512_rol_epi64(z11, 21); + z15 = _mm512_rol_epi64(z15, 8); + z12 = _mm512_rol_epi64(z12, 18); + z16 = _mm512_rol_epi64(z16, 2); + z20 = _mm512_rol_epi64(z20, 61); + z4 = _mm512_rol_epi64(z4, 56); + z8 = _mm512_rol_epi64(z8, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z22; + z0 = _mm512_ternarylogic_epi64(z0, z22, z19, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z19, z11, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z11, z8, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z17; + z31 = z14; + z17 = _mm512_ternarylogic_epi64(z17, z14, z6, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z6, z3, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z3, z20, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z9; + z31 = z1; + z9 = _mm512_ternarylogic_epi64(z9, z1, z23, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z23, z15, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z15, z12, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z21; + z31 = z18; + z21 = _mm512_ternarylogic_epi64(z21, z18, z10, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z10, z7, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z7, z4, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z13; + z31 = z5; + z13 = _mm512_ternarylogic_epi64(z13, z5, z2, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z2, z24, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z24, z16, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 128))); + /* Round 3 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z17, z9, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z21, z13, 0x96); + z26 = z22; + z26 = _mm512_ternarylogic_epi64(z26, z14, z1, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z18, z5, 0x96); + z27 = z19; + z27 = _mm512_ternarylogic_epi64(z27, z6, z23, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z10, z2, 0x96); + z28 = z11; + z28 = _mm512_ternarylogic_epi64(z28, z3, z15, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z7, z24, 0x96); + z29 = z8; + z29 = _mm512_ternarylogic_epi64(z29, z20, z12, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z4, z16, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z17 = _mm512_xor_si512(z17, z30); + z9 = _mm512_xor_si512(z9, z30); + z21 = _mm512_xor_si512(z21, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z22 = _mm512_xor_si512(z22, z30); + z14 = _mm512_xor_si512(z14, z30); + z1 = _mm512_xor_si512(z1, z30); + z18 = _mm512_xor_si512(z18, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z19 = _mm512_xor_si512(z19, z30); + z6 = _mm512_xor_si512(z6, z30); + z23 = _mm512_xor_si512(z23, z30); + z10 = _mm512_xor_si512(z10, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z11 = _mm512_xor_si512(z11, z30); + z3 = _mm512_xor_si512(z3, z30); + z15 = _mm512_xor_si512(z15, z30); + z7 = _mm512_xor_si512(z7, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z8 = _mm512_xor_si512(z8, z30); + z20 = _mm512_xor_si512(z20, z30); + z12 = _mm512_xor_si512(z12, z30); + z4 = _mm512_xor_si512(z4, z30); + z16 = _mm512_xor_si512(z16, z30); + /* Rho - rotate each lane in place */ + z22 = _mm512_rol_epi64(z22, 1); + z19 = _mm512_rol_epi64(z19, 62); + z11 = _mm512_rol_epi64(z11, 28); + z8 = _mm512_rol_epi64(z8, 27); + z17 = _mm512_rol_epi64(z17, 36); + z14 = _mm512_rol_epi64(z14, 44); + z6 = _mm512_rol_epi64(z6, 6); + z3 = _mm512_rol_epi64(z3, 55); + z20 = _mm512_rol_epi64(z20, 20); + z9 = _mm512_rol_epi64(z9, 3); + z1 = _mm512_rol_epi64(z1, 10); + z23 = _mm512_rol_epi64(z23, 43); + z15 = _mm512_rol_epi64(z15, 25); + z12 = _mm512_rol_epi64(z12, 39); + z21 = _mm512_rol_epi64(z21, 41); + z18 = _mm512_rol_epi64(z18, 45); + z10 = _mm512_rol_epi64(z10, 15); + z7 = _mm512_rol_epi64(z7, 21); + z4 = _mm512_rol_epi64(z4, 8); + z13 = _mm512_rol_epi64(z13, 18); + z5 = _mm512_rol_epi64(z5, 2); + z2 = _mm512_rol_epi64(z2, 61); + z24 = _mm512_rol_epi64(z24, 56); + z16 = _mm512_rol_epi64(z16, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z14; + z0 = _mm512_ternarylogic_epi64(z0, z14, z23, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z23, z7, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z7, z16, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z11; + z31 = z20; + z11 = _mm512_ternarylogic_epi64(z11, z20, z9, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z9, z18, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z18, z2, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z22; + z31 = z6; + z22 = _mm512_ternarylogic_epi64(z22, z6, z15, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z15, z4, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z4, z13, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z8; + z31 = z17; + z8 = _mm512_ternarylogic_epi64(z8, z17, z1, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z1, z10, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z10, z24, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z19; + z31 = z3; + z19 = _mm512_ternarylogic_epi64(z19, z3, z12, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z12, z21, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z21, z5, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 192))); + /* Round 4 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z11, z22, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z8, z19, 0x96); + z26 = z14; + z26 = _mm512_ternarylogic_epi64(z26, z20, z6, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z17, z3, 0x96); + z27 = z23; + z27 = _mm512_ternarylogic_epi64(z27, z9, z15, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z1, z12, 0x96); + z28 = z7; + z28 = _mm512_ternarylogic_epi64(z28, z18, z4, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z10, z21, 0x96); + z29 = z16; + z29 = _mm512_ternarylogic_epi64(z29, z2, z13, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z24, z5, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z11 = _mm512_xor_si512(z11, z30); + z22 = _mm512_xor_si512(z22, z30); + z8 = _mm512_xor_si512(z8, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z14 = _mm512_xor_si512(z14, z30); + z20 = _mm512_xor_si512(z20, z30); + z6 = _mm512_xor_si512(z6, z30); + z17 = _mm512_xor_si512(z17, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z23 = _mm512_xor_si512(z23, z30); + z9 = _mm512_xor_si512(z9, z30); + z15 = _mm512_xor_si512(z15, z30); + z1 = _mm512_xor_si512(z1, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z7 = _mm512_xor_si512(z7, z30); + z18 = _mm512_xor_si512(z18, z30); + z4 = _mm512_xor_si512(z4, z30); + z10 = _mm512_xor_si512(z10, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z16 = _mm512_xor_si512(z16, z30); + z2 = _mm512_xor_si512(z2, z30); + z13 = _mm512_xor_si512(z13, z30); + z24 = _mm512_xor_si512(z24, z30); + z5 = _mm512_xor_si512(z5, z30); + /* Rho - rotate each lane in place */ + z14 = _mm512_rol_epi64(z14, 1); + z23 = _mm512_rol_epi64(z23, 62); + z7 = _mm512_rol_epi64(z7, 28); + z16 = _mm512_rol_epi64(z16, 27); + z11 = _mm512_rol_epi64(z11, 36); + z20 = _mm512_rol_epi64(z20, 44); + z9 = _mm512_rol_epi64(z9, 6); + z18 = _mm512_rol_epi64(z18, 55); + z2 = _mm512_rol_epi64(z2, 20); + z22 = _mm512_rol_epi64(z22, 3); + z6 = _mm512_rol_epi64(z6, 10); + z15 = _mm512_rol_epi64(z15, 43); + z4 = _mm512_rol_epi64(z4, 25); + z13 = _mm512_rol_epi64(z13, 39); + z8 = _mm512_rol_epi64(z8, 41); + z17 = _mm512_rol_epi64(z17, 45); + z1 = _mm512_rol_epi64(z1, 15); + z10 = _mm512_rol_epi64(z10, 21); + z24 = _mm512_rol_epi64(z24, 8); + z19 = _mm512_rol_epi64(z19, 18); + z3 = _mm512_rol_epi64(z3, 2); + z12 = _mm512_rol_epi64(z12, 61); + z21 = _mm512_rol_epi64(z21, 56); + z5 = _mm512_rol_epi64(z5, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z20; + z0 = _mm512_ternarylogic_epi64(z0, z20, z15, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z15, z10, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z10, z5, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z7; + z31 = z2; + z7 = _mm512_ternarylogic_epi64(z7, z2, z22, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z22, z17, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z17, z12, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z14; + z31 = z9; + z14 = _mm512_ternarylogic_epi64(z14, z9, z4, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z4, z24, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z24, z19, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z16; + z31 = z11; + z16 = _mm512_ternarylogic_epi64(z16, z11, z6, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z6, z1, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z1, z21, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z23; + z31 = z18; + z23 = _mm512_ternarylogic_epi64(z23, z18, z13, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z13, z8, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z8, z3, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 256))); + /* Round 5 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z7, z14, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z16, z23, 0x96); + z26 = z20; + z26 = _mm512_ternarylogic_epi64(z26, z2, z9, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z11, z18, 0x96); + z27 = z15; + z27 = _mm512_ternarylogic_epi64(z27, z22, z4, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z6, z13, 0x96); + z28 = z10; + z28 = _mm512_ternarylogic_epi64(z28, z17, z24, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z1, z8, 0x96); + z29 = z5; + z29 = _mm512_ternarylogic_epi64(z29, z12, z19, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z21, z3, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z7 = _mm512_xor_si512(z7, z30); + z14 = _mm512_xor_si512(z14, z30); + z16 = _mm512_xor_si512(z16, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z20 = _mm512_xor_si512(z20, z30); + z2 = _mm512_xor_si512(z2, z30); + z9 = _mm512_xor_si512(z9, z30); + z11 = _mm512_xor_si512(z11, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z15 = _mm512_xor_si512(z15, z30); + z22 = _mm512_xor_si512(z22, z30); + z4 = _mm512_xor_si512(z4, z30); + z6 = _mm512_xor_si512(z6, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z10 = _mm512_xor_si512(z10, z30); + z17 = _mm512_xor_si512(z17, z30); + z24 = _mm512_xor_si512(z24, z30); + z1 = _mm512_xor_si512(z1, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z5 = _mm512_xor_si512(z5, z30); + z12 = _mm512_xor_si512(z12, z30); + z19 = _mm512_xor_si512(z19, z30); + z21 = _mm512_xor_si512(z21, z30); + z3 = _mm512_xor_si512(z3, z30); + /* Rho - rotate each lane in place */ + z20 = _mm512_rol_epi64(z20, 1); + z15 = _mm512_rol_epi64(z15, 62); + z10 = _mm512_rol_epi64(z10, 28); + z5 = _mm512_rol_epi64(z5, 27); + z7 = _mm512_rol_epi64(z7, 36); + z2 = _mm512_rol_epi64(z2, 44); + z22 = _mm512_rol_epi64(z22, 6); + z17 = _mm512_rol_epi64(z17, 55); + z12 = _mm512_rol_epi64(z12, 20); + z14 = _mm512_rol_epi64(z14, 3); + z9 = _mm512_rol_epi64(z9, 10); + z4 = _mm512_rol_epi64(z4, 43); + z24 = _mm512_rol_epi64(z24, 25); + z19 = _mm512_rol_epi64(z19, 39); + z16 = _mm512_rol_epi64(z16, 41); + z11 = _mm512_rol_epi64(z11, 45); + z6 = _mm512_rol_epi64(z6, 15); + z1 = _mm512_rol_epi64(z1, 21); + z21 = _mm512_rol_epi64(z21, 8); + z23 = _mm512_rol_epi64(z23, 18); + z18 = _mm512_rol_epi64(z18, 2); + z13 = _mm512_rol_epi64(z13, 61); + z8 = _mm512_rol_epi64(z8, 56); + z3 = _mm512_rol_epi64(z3, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z2; + z0 = _mm512_ternarylogic_epi64(z0, z2, z4, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z4, z1, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z1, z3, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z10; + z31 = z12; + z10 = _mm512_ternarylogic_epi64(z10, z12, z14, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z14, z11, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z11, z13, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z20; + z31 = z22; + z20 = _mm512_ternarylogic_epi64(z20, z22, z24, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z24, z21, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z21, z23, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z5; + z31 = z7; + z5 = _mm512_ternarylogic_epi64(z5, z7, z9, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z9, z6, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z6, z8, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z15; + z31 = z17; + z15 = _mm512_ternarylogic_epi64(z15, z17, z19, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z19, z16, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z16, z18, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 320))); + /* Round 6 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z10, z20, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z5, z15, 0x96); + z26 = z2; + z26 = _mm512_ternarylogic_epi64(z26, z12, z22, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z7, z17, 0x96); + z27 = z4; + z27 = _mm512_ternarylogic_epi64(z27, z14, z24, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z9, z19, 0x96); + z28 = z1; + z28 = _mm512_ternarylogic_epi64(z28, z11, z21, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z6, z16, 0x96); + z29 = z3; + z29 = _mm512_ternarylogic_epi64(z29, z13, z23, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z8, z18, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z10 = _mm512_xor_si512(z10, z30); + z20 = _mm512_xor_si512(z20, z30); + z5 = _mm512_xor_si512(z5, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z2 = _mm512_xor_si512(z2, z30); + z12 = _mm512_xor_si512(z12, z30); + z22 = _mm512_xor_si512(z22, z30); + z7 = _mm512_xor_si512(z7, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z4 = _mm512_xor_si512(z4, z30); + z14 = _mm512_xor_si512(z14, z30); + z24 = _mm512_xor_si512(z24, z30); + z9 = _mm512_xor_si512(z9, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z1 = _mm512_xor_si512(z1, z30); + z11 = _mm512_xor_si512(z11, z30); + z21 = _mm512_xor_si512(z21, z30); + z6 = _mm512_xor_si512(z6, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z3 = _mm512_xor_si512(z3, z30); + z13 = _mm512_xor_si512(z13, z30); + z23 = _mm512_xor_si512(z23, z30); + z8 = _mm512_xor_si512(z8, z30); + z18 = _mm512_xor_si512(z18, z30); + /* Rho - rotate each lane in place */ + z2 = _mm512_rol_epi64(z2, 1); + z4 = _mm512_rol_epi64(z4, 62); + z1 = _mm512_rol_epi64(z1, 28); + z3 = _mm512_rol_epi64(z3, 27); + z10 = _mm512_rol_epi64(z10, 36); + z12 = _mm512_rol_epi64(z12, 44); + z14 = _mm512_rol_epi64(z14, 6); + z11 = _mm512_rol_epi64(z11, 55); + z13 = _mm512_rol_epi64(z13, 20); + z20 = _mm512_rol_epi64(z20, 3); + z22 = _mm512_rol_epi64(z22, 10); + z24 = _mm512_rol_epi64(z24, 43); + z21 = _mm512_rol_epi64(z21, 25); + z23 = _mm512_rol_epi64(z23, 39); + z5 = _mm512_rol_epi64(z5, 41); + z7 = _mm512_rol_epi64(z7, 45); + z9 = _mm512_rol_epi64(z9, 15); + z6 = _mm512_rol_epi64(z6, 21); + z8 = _mm512_rol_epi64(z8, 8); + z15 = _mm512_rol_epi64(z15, 18); + z17 = _mm512_rol_epi64(z17, 2); + z19 = _mm512_rol_epi64(z19, 61); + z16 = _mm512_rol_epi64(z16, 56); + z18 = _mm512_rol_epi64(z18, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z12; + z0 = _mm512_ternarylogic_epi64(z0, z12, z24, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z24, z6, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z6, z18, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z1; + z31 = z13; + z1 = _mm512_ternarylogic_epi64(z1, z13, z20, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z20, z7, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z7, z19, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z2; + z31 = z14; + z2 = _mm512_ternarylogic_epi64(z2, z14, z21, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z21, z8, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z8, z15, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z3; + z31 = z10; + z3 = _mm512_ternarylogic_epi64(z3, z10, z22, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z22, z9, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z9, z16, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z4; + z31 = z11; + z4 = _mm512_ternarylogic_epi64(z4, z11, z23, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z23, z5, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z5, z17, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 384))); + /* Round 7 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z1, z2, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z3, z4, 0x96); + z26 = z12; + z26 = _mm512_ternarylogic_epi64(z26, z13, z14, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z10, z11, 0x96); + z27 = z24; + z27 = _mm512_ternarylogic_epi64(z27, z20, z21, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z22, z23, 0x96); + z28 = z6; + z28 = _mm512_ternarylogic_epi64(z28, z7, z8, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z9, z5, 0x96); + z29 = z18; + z29 = _mm512_ternarylogic_epi64(z29, z19, z15, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z16, z17, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z1 = _mm512_xor_si512(z1, z30); + z2 = _mm512_xor_si512(z2, z30); + z3 = _mm512_xor_si512(z3, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z12 = _mm512_xor_si512(z12, z30); + z13 = _mm512_xor_si512(z13, z30); + z14 = _mm512_xor_si512(z14, z30); + z10 = _mm512_xor_si512(z10, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z24 = _mm512_xor_si512(z24, z30); + z20 = _mm512_xor_si512(z20, z30); + z21 = _mm512_xor_si512(z21, z30); + z22 = _mm512_xor_si512(z22, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z6 = _mm512_xor_si512(z6, z30); + z7 = _mm512_xor_si512(z7, z30); + z8 = _mm512_xor_si512(z8, z30); + z9 = _mm512_xor_si512(z9, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z18 = _mm512_xor_si512(z18, z30); + z19 = _mm512_xor_si512(z19, z30); + z15 = _mm512_xor_si512(z15, z30); + z16 = _mm512_xor_si512(z16, z30); + z17 = _mm512_xor_si512(z17, z30); + /* Rho - rotate each lane in place */ + z12 = _mm512_rol_epi64(z12, 1); + z24 = _mm512_rol_epi64(z24, 62); + z6 = _mm512_rol_epi64(z6, 28); + z18 = _mm512_rol_epi64(z18, 27); + z1 = _mm512_rol_epi64(z1, 36); + z13 = _mm512_rol_epi64(z13, 44); + z20 = _mm512_rol_epi64(z20, 6); + z7 = _mm512_rol_epi64(z7, 55); + z19 = _mm512_rol_epi64(z19, 20); + z2 = _mm512_rol_epi64(z2, 3); + z14 = _mm512_rol_epi64(z14, 10); + z21 = _mm512_rol_epi64(z21, 43); + z8 = _mm512_rol_epi64(z8, 25); + z15 = _mm512_rol_epi64(z15, 39); + z3 = _mm512_rol_epi64(z3, 41); + z10 = _mm512_rol_epi64(z10, 45); + z22 = _mm512_rol_epi64(z22, 15); + z9 = _mm512_rol_epi64(z9, 21); + z16 = _mm512_rol_epi64(z16, 8); + z4 = _mm512_rol_epi64(z4, 18); + z11 = _mm512_rol_epi64(z11, 2); + z23 = _mm512_rol_epi64(z23, 61); + z5 = _mm512_rol_epi64(z5, 56); + z17 = _mm512_rol_epi64(z17, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z13; + z0 = _mm512_ternarylogic_epi64(z0, z13, z21, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z21, z9, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z9, z17, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z6; + z31 = z19; + z6 = _mm512_ternarylogic_epi64(z6, z19, z2, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z2, z10, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z10, z23, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z12; + z31 = z20; + z12 = _mm512_ternarylogic_epi64(z12, z20, z8, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z8, z16, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z16, z4, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z18; + z31 = z1; + z18 = _mm512_ternarylogic_epi64(z18, z1, z14, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z14, z22, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z22, z5, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z24; + z31 = z7; + z24 = _mm512_ternarylogic_epi64(z24, z7, z15, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z15, z3, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z3, z11, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 448))); + /* Round 8 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z6, z12, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z18, z24, 0x96); + z26 = z13; + z26 = _mm512_ternarylogic_epi64(z26, z19, z20, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z1, z7, 0x96); + z27 = z21; + z27 = _mm512_ternarylogic_epi64(z27, z2, z8, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z14, z15, 0x96); + z28 = z9; + z28 = _mm512_ternarylogic_epi64(z28, z10, z16, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z22, z3, 0x96); + z29 = z17; + z29 = _mm512_ternarylogic_epi64(z29, z23, z4, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z5, z11, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z6 = _mm512_xor_si512(z6, z30); + z12 = _mm512_xor_si512(z12, z30); + z18 = _mm512_xor_si512(z18, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z13 = _mm512_xor_si512(z13, z30); + z19 = _mm512_xor_si512(z19, z30); + z20 = _mm512_xor_si512(z20, z30); + z1 = _mm512_xor_si512(z1, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z21 = _mm512_xor_si512(z21, z30); + z2 = _mm512_xor_si512(z2, z30); + z8 = _mm512_xor_si512(z8, z30); + z14 = _mm512_xor_si512(z14, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z9 = _mm512_xor_si512(z9, z30); + z10 = _mm512_xor_si512(z10, z30); + z16 = _mm512_xor_si512(z16, z30); + z22 = _mm512_xor_si512(z22, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z17 = _mm512_xor_si512(z17, z30); + z23 = _mm512_xor_si512(z23, z30); + z4 = _mm512_xor_si512(z4, z30); + z5 = _mm512_xor_si512(z5, z30); + z11 = _mm512_xor_si512(z11, z30); + /* Rho - rotate each lane in place */ + z13 = _mm512_rol_epi64(z13, 1); + z21 = _mm512_rol_epi64(z21, 62); + z9 = _mm512_rol_epi64(z9, 28); + z17 = _mm512_rol_epi64(z17, 27); + z6 = _mm512_rol_epi64(z6, 36); + z19 = _mm512_rol_epi64(z19, 44); + z2 = _mm512_rol_epi64(z2, 6); + z10 = _mm512_rol_epi64(z10, 55); + z23 = _mm512_rol_epi64(z23, 20); + z12 = _mm512_rol_epi64(z12, 3); + z20 = _mm512_rol_epi64(z20, 10); + z8 = _mm512_rol_epi64(z8, 43); + z16 = _mm512_rol_epi64(z16, 25); + z4 = _mm512_rol_epi64(z4, 39); + z18 = _mm512_rol_epi64(z18, 41); + z1 = _mm512_rol_epi64(z1, 45); + z14 = _mm512_rol_epi64(z14, 15); + z22 = _mm512_rol_epi64(z22, 21); + z5 = _mm512_rol_epi64(z5, 8); + z24 = _mm512_rol_epi64(z24, 18); + z7 = _mm512_rol_epi64(z7, 2); + z15 = _mm512_rol_epi64(z15, 61); + z3 = _mm512_rol_epi64(z3, 56); + z11 = _mm512_rol_epi64(z11, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z19; + z0 = _mm512_ternarylogic_epi64(z0, z19, z8, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z8, z22, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z22, z11, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z9; + z31 = z23; + z9 = _mm512_ternarylogic_epi64(z9, z23, z12, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z12, z1, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z1, z15, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z13; + z31 = z2; + z13 = _mm512_ternarylogic_epi64(z13, z2, z16, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z16, z5, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z5, z24, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z17; + z31 = z6; + z17 = _mm512_ternarylogic_epi64(z17, z6, z20, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z20, z14, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z14, z3, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z21; + z31 = z10; + z21 = _mm512_ternarylogic_epi64(z21, z10, z4, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z4, z18, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z18, z7, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 512))); + /* Round 9 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z9, z13, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z17, z21, 0x96); + z26 = z19; + z26 = _mm512_ternarylogic_epi64(z26, z23, z2, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z6, z10, 0x96); + z27 = z8; + z27 = _mm512_ternarylogic_epi64(z27, z12, z16, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z20, z4, 0x96); + z28 = z22; + z28 = _mm512_ternarylogic_epi64(z28, z1, z5, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z14, z18, 0x96); + z29 = z11; + z29 = _mm512_ternarylogic_epi64(z29, z15, z24, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z3, z7, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z9 = _mm512_xor_si512(z9, z30); + z13 = _mm512_xor_si512(z13, z30); + z17 = _mm512_xor_si512(z17, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z19 = _mm512_xor_si512(z19, z30); + z23 = _mm512_xor_si512(z23, z30); + z2 = _mm512_xor_si512(z2, z30); + z6 = _mm512_xor_si512(z6, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z8 = _mm512_xor_si512(z8, z30); + z12 = _mm512_xor_si512(z12, z30); + z16 = _mm512_xor_si512(z16, z30); + z20 = _mm512_xor_si512(z20, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z22 = _mm512_xor_si512(z22, z30); + z1 = _mm512_xor_si512(z1, z30); + z5 = _mm512_xor_si512(z5, z30); + z14 = _mm512_xor_si512(z14, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z11 = _mm512_xor_si512(z11, z30); + z15 = _mm512_xor_si512(z15, z30); + z24 = _mm512_xor_si512(z24, z30); + z3 = _mm512_xor_si512(z3, z30); + z7 = _mm512_xor_si512(z7, z30); + /* Rho - rotate each lane in place */ + z19 = _mm512_rol_epi64(z19, 1); + z8 = _mm512_rol_epi64(z8, 62); + z22 = _mm512_rol_epi64(z22, 28); + z11 = _mm512_rol_epi64(z11, 27); + z9 = _mm512_rol_epi64(z9, 36); + z23 = _mm512_rol_epi64(z23, 44); + z12 = _mm512_rol_epi64(z12, 6); + z1 = _mm512_rol_epi64(z1, 55); + z15 = _mm512_rol_epi64(z15, 20); + z13 = _mm512_rol_epi64(z13, 3); + z2 = _mm512_rol_epi64(z2, 10); + z16 = _mm512_rol_epi64(z16, 43); + z5 = _mm512_rol_epi64(z5, 25); + z24 = _mm512_rol_epi64(z24, 39); + z17 = _mm512_rol_epi64(z17, 41); + z6 = _mm512_rol_epi64(z6, 45); + z20 = _mm512_rol_epi64(z20, 15); + z14 = _mm512_rol_epi64(z14, 21); + z3 = _mm512_rol_epi64(z3, 8); + z21 = _mm512_rol_epi64(z21, 18); + z10 = _mm512_rol_epi64(z10, 2); + z4 = _mm512_rol_epi64(z4, 61); + z18 = _mm512_rol_epi64(z18, 56); + z7 = _mm512_rol_epi64(z7, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z23; + z0 = _mm512_ternarylogic_epi64(z0, z23, z16, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z16, z14, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z14, z7, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z22; + z31 = z15; + z22 = _mm512_ternarylogic_epi64(z22, z15, z13, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z13, z6, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z6, z4, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z19; + z31 = z12; + z19 = _mm512_ternarylogic_epi64(z19, z12, z5, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z5, z3, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z3, z21, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z11; + z31 = z9; + z11 = _mm512_ternarylogic_epi64(z11, z9, z2, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z2, z20, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z20, z18, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z8; + z31 = z1; + z8 = _mm512_ternarylogic_epi64(z8, z1, z24, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z24, z17, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z17, z10, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 576))); + /* Round 10 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z22, z19, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z11, z8, 0x96); + z26 = z23; + z26 = _mm512_ternarylogic_epi64(z26, z15, z12, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z9, z1, 0x96); + z27 = z16; + z27 = _mm512_ternarylogic_epi64(z27, z13, z5, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z2, z24, 0x96); + z28 = z14; + z28 = _mm512_ternarylogic_epi64(z28, z6, z3, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z20, z17, 0x96); + z29 = z7; + z29 = _mm512_ternarylogic_epi64(z29, z4, z21, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z18, z10, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z22 = _mm512_xor_si512(z22, z30); + z19 = _mm512_xor_si512(z19, z30); + z11 = _mm512_xor_si512(z11, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z23 = _mm512_xor_si512(z23, z30); + z15 = _mm512_xor_si512(z15, z30); + z12 = _mm512_xor_si512(z12, z30); + z9 = _mm512_xor_si512(z9, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z16 = _mm512_xor_si512(z16, z30); + z13 = _mm512_xor_si512(z13, z30); + z5 = _mm512_xor_si512(z5, z30); + z2 = _mm512_xor_si512(z2, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z14 = _mm512_xor_si512(z14, z30); + z6 = _mm512_xor_si512(z6, z30); + z3 = _mm512_xor_si512(z3, z30); + z20 = _mm512_xor_si512(z20, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z7 = _mm512_xor_si512(z7, z30); + z4 = _mm512_xor_si512(z4, z30); + z21 = _mm512_xor_si512(z21, z30); + z18 = _mm512_xor_si512(z18, z30); + z10 = _mm512_xor_si512(z10, z30); + /* Rho - rotate each lane in place */ + z23 = _mm512_rol_epi64(z23, 1); + z16 = _mm512_rol_epi64(z16, 62); + z14 = _mm512_rol_epi64(z14, 28); + z7 = _mm512_rol_epi64(z7, 27); + z22 = _mm512_rol_epi64(z22, 36); + z15 = _mm512_rol_epi64(z15, 44); + z13 = _mm512_rol_epi64(z13, 6); + z6 = _mm512_rol_epi64(z6, 55); + z4 = _mm512_rol_epi64(z4, 20); + z19 = _mm512_rol_epi64(z19, 3); + z12 = _mm512_rol_epi64(z12, 10); + z5 = _mm512_rol_epi64(z5, 43); + z3 = _mm512_rol_epi64(z3, 25); + z21 = _mm512_rol_epi64(z21, 39); + z11 = _mm512_rol_epi64(z11, 41); + z9 = _mm512_rol_epi64(z9, 45); + z2 = _mm512_rol_epi64(z2, 15); + z20 = _mm512_rol_epi64(z20, 21); + z18 = _mm512_rol_epi64(z18, 8); + z8 = _mm512_rol_epi64(z8, 18); + z1 = _mm512_rol_epi64(z1, 2); + z24 = _mm512_rol_epi64(z24, 61); + z17 = _mm512_rol_epi64(z17, 56); + z10 = _mm512_rol_epi64(z10, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z15; + z0 = _mm512_ternarylogic_epi64(z0, z15, z5, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z5, z20, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z20, z10, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z14; + z31 = z4; + z14 = _mm512_ternarylogic_epi64(z14, z4, z19, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z19, z9, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z9, z24, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z23; + z31 = z13; + z23 = _mm512_ternarylogic_epi64(z23, z13, z3, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z3, z18, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z18, z8, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z7; + z31 = z22; + z7 = _mm512_ternarylogic_epi64(z7, z22, z12, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z12, z2, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z2, z17, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z16; + z31 = z6; + z16 = _mm512_ternarylogic_epi64(z16, z6, z21, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z21, z11, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z11, z1, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 640))); + /* Round 11 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z14, z23, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z7, z16, 0x96); + z26 = z15; + z26 = _mm512_ternarylogic_epi64(z26, z4, z13, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z22, z6, 0x96); + z27 = z5; + z27 = _mm512_ternarylogic_epi64(z27, z19, z3, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z12, z21, 0x96); + z28 = z20; + z28 = _mm512_ternarylogic_epi64(z28, z9, z18, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z2, z11, 0x96); + z29 = z10; + z29 = _mm512_ternarylogic_epi64(z29, z24, z8, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z17, z1, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z14 = _mm512_xor_si512(z14, z30); + z23 = _mm512_xor_si512(z23, z30); + z7 = _mm512_xor_si512(z7, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z15 = _mm512_xor_si512(z15, z30); + z4 = _mm512_xor_si512(z4, z30); + z13 = _mm512_xor_si512(z13, z30); + z22 = _mm512_xor_si512(z22, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z5 = _mm512_xor_si512(z5, z30); + z19 = _mm512_xor_si512(z19, z30); + z3 = _mm512_xor_si512(z3, z30); + z12 = _mm512_xor_si512(z12, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z20 = _mm512_xor_si512(z20, z30); + z9 = _mm512_xor_si512(z9, z30); + z18 = _mm512_xor_si512(z18, z30); + z2 = _mm512_xor_si512(z2, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z10 = _mm512_xor_si512(z10, z30); + z24 = _mm512_xor_si512(z24, z30); + z8 = _mm512_xor_si512(z8, z30); + z17 = _mm512_xor_si512(z17, z30); + z1 = _mm512_xor_si512(z1, z30); + /* Rho - rotate each lane in place */ + z15 = _mm512_rol_epi64(z15, 1); + z5 = _mm512_rol_epi64(z5, 62); + z20 = _mm512_rol_epi64(z20, 28); + z10 = _mm512_rol_epi64(z10, 27); + z14 = _mm512_rol_epi64(z14, 36); + z4 = _mm512_rol_epi64(z4, 44); + z19 = _mm512_rol_epi64(z19, 6); + z9 = _mm512_rol_epi64(z9, 55); + z24 = _mm512_rol_epi64(z24, 20); + z23 = _mm512_rol_epi64(z23, 3); + z13 = _mm512_rol_epi64(z13, 10); + z3 = _mm512_rol_epi64(z3, 43); + z18 = _mm512_rol_epi64(z18, 25); + z8 = _mm512_rol_epi64(z8, 39); + z7 = _mm512_rol_epi64(z7, 41); + z22 = _mm512_rol_epi64(z22, 45); + z12 = _mm512_rol_epi64(z12, 15); + z2 = _mm512_rol_epi64(z2, 21); + z17 = _mm512_rol_epi64(z17, 8); + z16 = _mm512_rol_epi64(z16, 18); + z6 = _mm512_rol_epi64(z6, 2); + z21 = _mm512_rol_epi64(z21, 61); + z11 = _mm512_rol_epi64(z11, 56); + z1 = _mm512_rol_epi64(z1, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z4; + z0 = _mm512_ternarylogic_epi64(z0, z4, z3, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z3, z2, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z2, z1, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z20; + z31 = z24; + z20 = _mm512_ternarylogic_epi64(z20, z24, z23, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z23, z22, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z22, z21, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z15; + z31 = z19; + z15 = _mm512_ternarylogic_epi64(z15, z19, z18, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z18, z17, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z17, z16, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z10; + z31 = z14; + z10 = _mm512_ternarylogic_epi64(z10, z14, z13, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z13, z12, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z12, z11, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z5; + z31 = z9; + z5 = _mm512_ternarylogic_epi64(z5, z9, z8, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z8, z7, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z7, z6, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 704))); + /* Round 12 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z20, z15, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z10, z5, 0x96); + z26 = z4; + z26 = _mm512_ternarylogic_epi64(z26, z24, z19, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z14, z9, 0x96); + z27 = z3; + z27 = _mm512_ternarylogic_epi64(z27, z23, z18, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z13, z8, 0x96); + z28 = z2; + z28 = _mm512_ternarylogic_epi64(z28, z22, z17, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z12, z7, 0x96); + z29 = z1; + z29 = _mm512_ternarylogic_epi64(z29, z21, z16, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z11, z6, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z20 = _mm512_xor_si512(z20, z30); + z15 = _mm512_xor_si512(z15, z30); + z10 = _mm512_xor_si512(z10, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z4 = _mm512_xor_si512(z4, z30); + z24 = _mm512_xor_si512(z24, z30); + z19 = _mm512_xor_si512(z19, z30); + z14 = _mm512_xor_si512(z14, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z3 = _mm512_xor_si512(z3, z30); + z23 = _mm512_xor_si512(z23, z30); + z18 = _mm512_xor_si512(z18, z30); + z13 = _mm512_xor_si512(z13, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z2 = _mm512_xor_si512(z2, z30); + z22 = _mm512_xor_si512(z22, z30); + z17 = _mm512_xor_si512(z17, z30); + z12 = _mm512_xor_si512(z12, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z1 = _mm512_xor_si512(z1, z30); + z21 = _mm512_xor_si512(z21, z30); + z16 = _mm512_xor_si512(z16, z30); + z11 = _mm512_xor_si512(z11, z30); + z6 = _mm512_xor_si512(z6, z30); + /* Rho - rotate each lane in place */ + z4 = _mm512_rol_epi64(z4, 1); + z3 = _mm512_rol_epi64(z3, 62); + z2 = _mm512_rol_epi64(z2, 28); + z1 = _mm512_rol_epi64(z1, 27); + z20 = _mm512_rol_epi64(z20, 36); + z24 = _mm512_rol_epi64(z24, 44); + z23 = _mm512_rol_epi64(z23, 6); + z22 = _mm512_rol_epi64(z22, 55); + z21 = _mm512_rol_epi64(z21, 20); + z15 = _mm512_rol_epi64(z15, 3); + z19 = _mm512_rol_epi64(z19, 10); + z18 = _mm512_rol_epi64(z18, 43); + z17 = _mm512_rol_epi64(z17, 25); + z16 = _mm512_rol_epi64(z16, 39); + z10 = _mm512_rol_epi64(z10, 41); + z14 = _mm512_rol_epi64(z14, 45); + z13 = _mm512_rol_epi64(z13, 15); + z12 = _mm512_rol_epi64(z12, 21); + z11 = _mm512_rol_epi64(z11, 8); + z5 = _mm512_rol_epi64(z5, 18); + z9 = _mm512_rol_epi64(z9, 2); + z8 = _mm512_rol_epi64(z8, 61); + z7 = _mm512_rol_epi64(z7, 56); + z6 = _mm512_rol_epi64(z6, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z24; + z0 = _mm512_ternarylogic_epi64(z0, z24, z18, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z18, z12, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z12, z6, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z2; + z31 = z21; + z2 = _mm512_ternarylogic_epi64(z2, z21, z15, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z15, z14, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z14, z8, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z4; + z31 = z23; + z4 = _mm512_ternarylogic_epi64(z4, z23, z17, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z17, z11, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z11, z5, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z1; + z31 = z20; + z1 = _mm512_ternarylogic_epi64(z1, z20, z19, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z19, z13, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z13, z7, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z3; + z31 = z22; + z3 = _mm512_ternarylogic_epi64(z3, z22, z16, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z16, z10, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z10, z9, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 768))); + /* Round 13 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z2, z4, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z1, z3, 0x96); + z26 = z24; + z26 = _mm512_ternarylogic_epi64(z26, z21, z23, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z20, z22, 0x96); + z27 = z18; + z27 = _mm512_ternarylogic_epi64(z27, z15, z17, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z19, z16, 0x96); + z28 = z12; + z28 = _mm512_ternarylogic_epi64(z28, z14, z11, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z13, z10, 0x96); + z29 = z6; + z29 = _mm512_ternarylogic_epi64(z29, z8, z5, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z7, z9, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z2 = _mm512_xor_si512(z2, z30); + z4 = _mm512_xor_si512(z4, z30); + z1 = _mm512_xor_si512(z1, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z24 = _mm512_xor_si512(z24, z30); + z21 = _mm512_xor_si512(z21, z30); + z23 = _mm512_xor_si512(z23, z30); + z20 = _mm512_xor_si512(z20, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z18 = _mm512_xor_si512(z18, z30); + z15 = _mm512_xor_si512(z15, z30); + z17 = _mm512_xor_si512(z17, z30); + z19 = _mm512_xor_si512(z19, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z12 = _mm512_xor_si512(z12, z30); + z14 = _mm512_xor_si512(z14, z30); + z11 = _mm512_xor_si512(z11, z30); + z13 = _mm512_xor_si512(z13, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z6 = _mm512_xor_si512(z6, z30); + z8 = _mm512_xor_si512(z8, z30); + z5 = _mm512_xor_si512(z5, z30); + z7 = _mm512_xor_si512(z7, z30); + z9 = _mm512_xor_si512(z9, z30); + /* Rho - rotate each lane in place */ + z24 = _mm512_rol_epi64(z24, 1); + z18 = _mm512_rol_epi64(z18, 62); + z12 = _mm512_rol_epi64(z12, 28); + z6 = _mm512_rol_epi64(z6, 27); + z2 = _mm512_rol_epi64(z2, 36); + z21 = _mm512_rol_epi64(z21, 44); + z15 = _mm512_rol_epi64(z15, 6); + z14 = _mm512_rol_epi64(z14, 55); + z8 = _mm512_rol_epi64(z8, 20); + z4 = _mm512_rol_epi64(z4, 3); + z23 = _mm512_rol_epi64(z23, 10); + z17 = _mm512_rol_epi64(z17, 43); + z11 = _mm512_rol_epi64(z11, 25); + z5 = _mm512_rol_epi64(z5, 39); + z1 = _mm512_rol_epi64(z1, 41); + z20 = _mm512_rol_epi64(z20, 45); + z19 = _mm512_rol_epi64(z19, 15); + z13 = _mm512_rol_epi64(z13, 21); + z7 = _mm512_rol_epi64(z7, 8); + z3 = _mm512_rol_epi64(z3, 18); + z22 = _mm512_rol_epi64(z22, 2); + z16 = _mm512_rol_epi64(z16, 61); + z10 = _mm512_rol_epi64(z10, 56); + z9 = _mm512_rol_epi64(z9, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z21; + z0 = _mm512_ternarylogic_epi64(z0, z21, z17, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z17, z13, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z13, z9, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z12; + z31 = z8; + z12 = _mm512_ternarylogic_epi64(z12, z8, z4, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z4, z20, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z20, z16, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z24; + z31 = z15; + z24 = _mm512_ternarylogic_epi64(z24, z15, z11, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z11, z7, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z7, z3, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z6; + z31 = z2; + z6 = _mm512_ternarylogic_epi64(z6, z2, z23, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z23, z19, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z19, z10, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z18; + z31 = z14; + z18 = _mm512_ternarylogic_epi64(z18, z14, z5, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z5, z1, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z1, z22, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 832))); + /* Round 14 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z12, z24, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z6, z18, 0x96); + z26 = z21; + z26 = _mm512_ternarylogic_epi64(z26, z8, z15, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z2, z14, 0x96); + z27 = z17; + z27 = _mm512_ternarylogic_epi64(z27, z4, z11, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z23, z5, 0x96); + z28 = z13; + z28 = _mm512_ternarylogic_epi64(z28, z20, z7, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z19, z1, 0x96); + z29 = z9; + z29 = _mm512_ternarylogic_epi64(z29, z16, z3, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z10, z22, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z12 = _mm512_xor_si512(z12, z30); + z24 = _mm512_xor_si512(z24, z30); + z6 = _mm512_xor_si512(z6, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z21 = _mm512_xor_si512(z21, z30); + z8 = _mm512_xor_si512(z8, z30); + z15 = _mm512_xor_si512(z15, z30); + z2 = _mm512_xor_si512(z2, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z17 = _mm512_xor_si512(z17, z30); + z4 = _mm512_xor_si512(z4, z30); + z11 = _mm512_xor_si512(z11, z30); + z23 = _mm512_xor_si512(z23, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z13 = _mm512_xor_si512(z13, z30); + z20 = _mm512_xor_si512(z20, z30); + z7 = _mm512_xor_si512(z7, z30); + z19 = _mm512_xor_si512(z19, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z9 = _mm512_xor_si512(z9, z30); + z16 = _mm512_xor_si512(z16, z30); + z3 = _mm512_xor_si512(z3, z30); + z10 = _mm512_xor_si512(z10, z30); + z22 = _mm512_xor_si512(z22, z30); + /* Rho - rotate each lane in place */ + z21 = _mm512_rol_epi64(z21, 1); + z17 = _mm512_rol_epi64(z17, 62); + z13 = _mm512_rol_epi64(z13, 28); + z9 = _mm512_rol_epi64(z9, 27); + z12 = _mm512_rol_epi64(z12, 36); + z8 = _mm512_rol_epi64(z8, 44); + z4 = _mm512_rol_epi64(z4, 6); + z20 = _mm512_rol_epi64(z20, 55); + z16 = _mm512_rol_epi64(z16, 20); + z24 = _mm512_rol_epi64(z24, 3); + z15 = _mm512_rol_epi64(z15, 10); + z11 = _mm512_rol_epi64(z11, 43); + z7 = _mm512_rol_epi64(z7, 25); + z3 = _mm512_rol_epi64(z3, 39); + z6 = _mm512_rol_epi64(z6, 41); + z2 = _mm512_rol_epi64(z2, 45); + z23 = _mm512_rol_epi64(z23, 15); + z19 = _mm512_rol_epi64(z19, 21); + z10 = _mm512_rol_epi64(z10, 8); + z18 = _mm512_rol_epi64(z18, 18); + z14 = _mm512_rol_epi64(z14, 2); + z5 = _mm512_rol_epi64(z5, 61); + z1 = _mm512_rol_epi64(z1, 56); + z22 = _mm512_rol_epi64(z22, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z8; + z0 = _mm512_ternarylogic_epi64(z0, z8, z11, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z11, z19, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z19, z22, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z13; + z31 = z16; + z13 = _mm512_ternarylogic_epi64(z13, z16, z24, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z24, z2, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z2, z5, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z21; + z31 = z4; + z21 = _mm512_ternarylogic_epi64(z21, z4, z7, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z7, z10, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z10, z18, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z9; + z31 = z12; + z9 = _mm512_ternarylogic_epi64(z9, z12, z15, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z15, z23, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z23, z1, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z17; + z31 = z20; + z17 = _mm512_ternarylogic_epi64(z17, z20, z3, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z3, z6, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z6, z14, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 896))); + /* Round 15 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z13, z21, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z9, z17, 0x96); + z26 = z8; + z26 = _mm512_ternarylogic_epi64(z26, z16, z4, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z12, z20, 0x96); + z27 = z11; + z27 = _mm512_ternarylogic_epi64(z27, z24, z7, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z15, z3, 0x96); + z28 = z19; + z28 = _mm512_ternarylogic_epi64(z28, z2, z10, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z23, z6, 0x96); + z29 = z22; + z29 = _mm512_ternarylogic_epi64(z29, z5, z18, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z1, z14, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z13 = _mm512_xor_si512(z13, z30); + z21 = _mm512_xor_si512(z21, z30); + z9 = _mm512_xor_si512(z9, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z8 = _mm512_xor_si512(z8, z30); + z16 = _mm512_xor_si512(z16, z30); + z4 = _mm512_xor_si512(z4, z30); + z12 = _mm512_xor_si512(z12, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z11 = _mm512_xor_si512(z11, z30); + z24 = _mm512_xor_si512(z24, z30); + z7 = _mm512_xor_si512(z7, z30); + z15 = _mm512_xor_si512(z15, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z19 = _mm512_xor_si512(z19, z30); + z2 = _mm512_xor_si512(z2, z30); + z10 = _mm512_xor_si512(z10, z30); + z23 = _mm512_xor_si512(z23, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z22 = _mm512_xor_si512(z22, z30); + z5 = _mm512_xor_si512(z5, z30); + z18 = _mm512_xor_si512(z18, z30); + z1 = _mm512_xor_si512(z1, z30); + z14 = _mm512_xor_si512(z14, z30); + /* Rho - rotate each lane in place */ + z8 = _mm512_rol_epi64(z8, 1); + z11 = _mm512_rol_epi64(z11, 62); + z19 = _mm512_rol_epi64(z19, 28); + z22 = _mm512_rol_epi64(z22, 27); + z13 = _mm512_rol_epi64(z13, 36); + z16 = _mm512_rol_epi64(z16, 44); + z24 = _mm512_rol_epi64(z24, 6); + z2 = _mm512_rol_epi64(z2, 55); + z5 = _mm512_rol_epi64(z5, 20); + z21 = _mm512_rol_epi64(z21, 3); + z4 = _mm512_rol_epi64(z4, 10); + z7 = _mm512_rol_epi64(z7, 43); + z10 = _mm512_rol_epi64(z10, 25); + z18 = _mm512_rol_epi64(z18, 39); + z9 = _mm512_rol_epi64(z9, 41); + z12 = _mm512_rol_epi64(z12, 45); + z15 = _mm512_rol_epi64(z15, 15); + z23 = _mm512_rol_epi64(z23, 21); + z1 = _mm512_rol_epi64(z1, 8); + z17 = _mm512_rol_epi64(z17, 18); + z20 = _mm512_rol_epi64(z20, 2); + z3 = _mm512_rol_epi64(z3, 61); + z6 = _mm512_rol_epi64(z6, 56); + z14 = _mm512_rol_epi64(z14, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z16; + z0 = _mm512_ternarylogic_epi64(z0, z16, z7, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z7, z23, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z23, z14, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z19; + z31 = z5; + z19 = _mm512_ternarylogic_epi64(z19, z5, z21, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z21, z12, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z12, z3, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z8; + z31 = z24; + z8 = _mm512_ternarylogic_epi64(z8, z24, z10, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z10, z1, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z1, z17, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z22; + z31 = z13; + z22 = _mm512_ternarylogic_epi64(z22, z13, z4, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z4, z15, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z15, z6, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z11; + z31 = z2; + z11 = _mm512_ternarylogic_epi64(z11, z2, z18, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z18, z9, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z9, z20, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, 960))); + /* Round 16 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z19, z8, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z22, z11, 0x96); + z26 = z16; + z26 = _mm512_ternarylogic_epi64(z26, z5, z24, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z13, z2, 0x96); + z27 = z7; + z27 = _mm512_ternarylogic_epi64(z27, z21, z10, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z4, z18, 0x96); + z28 = z23; + z28 = _mm512_ternarylogic_epi64(z28, z12, z1, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z15, z9, 0x96); + z29 = z14; + z29 = _mm512_ternarylogic_epi64(z29, z3, z17, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z6, z20, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z19 = _mm512_xor_si512(z19, z30); + z8 = _mm512_xor_si512(z8, z30); + z22 = _mm512_xor_si512(z22, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z16 = _mm512_xor_si512(z16, z30); + z5 = _mm512_xor_si512(z5, z30); + z24 = _mm512_xor_si512(z24, z30); + z13 = _mm512_xor_si512(z13, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z7 = _mm512_xor_si512(z7, z30); + z21 = _mm512_xor_si512(z21, z30); + z10 = _mm512_xor_si512(z10, z30); + z4 = _mm512_xor_si512(z4, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z23 = _mm512_xor_si512(z23, z30); + z12 = _mm512_xor_si512(z12, z30); + z1 = _mm512_xor_si512(z1, z30); + z15 = _mm512_xor_si512(z15, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z14 = _mm512_xor_si512(z14, z30); + z3 = _mm512_xor_si512(z3, z30); + z17 = _mm512_xor_si512(z17, z30); + z6 = _mm512_xor_si512(z6, z30); + z20 = _mm512_xor_si512(z20, z30); + /* Rho - rotate each lane in place */ + z16 = _mm512_rol_epi64(z16, 1); + z7 = _mm512_rol_epi64(z7, 62); + z23 = _mm512_rol_epi64(z23, 28); + z14 = _mm512_rol_epi64(z14, 27); + z19 = _mm512_rol_epi64(z19, 36); + z5 = _mm512_rol_epi64(z5, 44); + z21 = _mm512_rol_epi64(z21, 6); + z12 = _mm512_rol_epi64(z12, 55); + z3 = _mm512_rol_epi64(z3, 20); + z8 = _mm512_rol_epi64(z8, 3); + z24 = _mm512_rol_epi64(z24, 10); + z10 = _mm512_rol_epi64(z10, 43); + z1 = _mm512_rol_epi64(z1, 25); + z17 = _mm512_rol_epi64(z17, 39); + z22 = _mm512_rol_epi64(z22, 41); + z13 = _mm512_rol_epi64(z13, 45); + z4 = _mm512_rol_epi64(z4, 15); + z15 = _mm512_rol_epi64(z15, 21); + z6 = _mm512_rol_epi64(z6, 8); + z11 = _mm512_rol_epi64(z11, 18); + z2 = _mm512_rol_epi64(z2, 2); + z18 = _mm512_rol_epi64(z18, 61); + z9 = _mm512_rol_epi64(z9, 56); + z20 = _mm512_rol_epi64(z20, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z5; + z0 = _mm512_ternarylogic_epi64(z0, z5, z10, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z10, z15, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z15, z20, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z23; + z31 = z3; + z23 = _mm512_ternarylogic_epi64(z23, z3, z8, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z8, z13, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z13, z18, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z16; + z31 = z21; + z16 = _mm512_ternarylogic_epi64(z16, z21, z1, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z1, z6, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z6, z11, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z14; + z31 = z19; + z14 = _mm512_ternarylogic_epi64(z14, z19, z24, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z24, z4, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z4, z9, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z7; + z31 = z12; + z7 = _mm512_ternarylogic_epi64(z7, z12, z17, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z17, z22, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z22, z2, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, + 1024))); + /* Round 17 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z23, z16, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z14, z7, 0x96); + z26 = z5; + z26 = _mm512_ternarylogic_epi64(z26, z3, z21, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z19, z12, 0x96); + z27 = z10; + z27 = _mm512_ternarylogic_epi64(z27, z8, z1, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z24, z17, 0x96); + z28 = z15; + z28 = _mm512_ternarylogic_epi64(z28, z13, z6, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z4, z22, 0x96); + z29 = z20; + z29 = _mm512_ternarylogic_epi64(z29, z18, z11, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z9, z2, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z23 = _mm512_xor_si512(z23, z30); + z16 = _mm512_xor_si512(z16, z30); + z14 = _mm512_xor_si512(z14, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z5 = _mm512_xor_si512(z5, z30); + z3 = _mm512_xor_si512(z3, z30); + z21 = _mm512_xor_si512(z21, z30); + z19 = _mm512_xor_si512(z19, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z10 = _mm512_xor_si512(z10, z30); + z8 = _mm512_xor_si512(z8, z30); + z1 = _mm512_xor_si512(z1, z30); + z24 = _mm512_xor_si512(z24, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z15 = _mm512_xor_si512(z15, z30); + z13 = _mm512_xor_si512(z13, z30); + z6 = _mm512_xor_si512(z6, z30); + z4 = _mm512_xor_si512(z4, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z20 = _mm512_xor_si512(z20, z30); + z18 = _mm512_xor_si512(z18, z30); + z11 = _mm512_xor_si512(z11, z30); + z9 = _mm512_xor_si512(z9, z30); + z2 = _mm512_xor_si512(z2, z30); + /* Rho - rotate each lane in place */ + z5 = _mm512_rol_epi64(z5, 1); + z10 = _mm512_rol_epi64(z10, 62); + z15 = _mm512_rol_epi64(z15, 28); + z20 = _mm512_rol_epi64(z20, 27); + z23 = _mm512_rol_epi64(z23, 36); + z3 = _mm512_rol_epi64(z3, 44); + z8 = _mm512_rol_epi64(z8, 6); + z13 = _mm512_rol_epi64(z13, 55); + z18 = _mm512_rol_epi64(z18, 20); + z16 = _mm512_rol_epi64(z16, 3); + z21 = _mm512_rol_epi64(z21, 10); + z1 = _mm512_rol_epi64(z1, 43); + z6 = _mm512_rol_epi64(z6, 25); + z11 = _mm512_rol_epi64(z11, 39); + z14 = _mm512_rol_epi64(z14, 41); + z19 = _mm512_rol_epi64(z19, 45); + z24 = _mm512_rol_epi64(z24, 15); + z4 = _mm512_rol_epi64(z4, 21); + z9 = _mm512_rol_epi64(z9, 8); + z7 = _mm512_rol_epi64(z7, 18); + z12 = _mm512_rol_epi64(z12, 2); + z17 = _mm512_rol_epi64(z17, 61); + z22 = _mm512_rol_epi64(z22, 56); + z2 = _mm512_rol_epi64(z2, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z3; + z0 = _mm512_ternarylogic_epi64(z0, z3, z1, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z1, z4, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z4, z2, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z15; + z31 = z18; + z15 = _mm512_ternarylogic_epi64(z15, z18, z16, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z16, z19, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z19, z17, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z5; + z31 = z8; + z5 = _mm512_ternarylogic_epi64(z5, z8, z6, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z6, z9, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z9, z7, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z20; + z31 = z23; + z20 = _mm512_ternarylogic_epi64(z20, z23, z21, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z21, z24, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z24, z22, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z10; + z31 = z13; + z10 = _mm512_ternarylogic_epi64(z10, z13, z11, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z11, z14, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z14, z12, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, + 1088))); + /* Round 18 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z15, z5, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z20, z10, 0x96); + z26 = z3; + z26 = _mm512_ternarylogic_epi64(z26, z18, z8, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z23, z13, 0x96); + z27 = z1; + z27 = _mm512_ternarylogic_epi64(z27, z16, z6, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z21, z11, 0x96); + z28 = z4; + z28 = _mm512_ternarylogic_epi64(z28, z19, z9, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z24, z14, 0x96); + z29 = z2; + z29 = _mm512_ternarylogic_epi64(z29, z17, z7, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z22, z12, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z15 = _mm512_xor_si512(z15, z30); + z5 = _mm512_xor_si512(z5, z30); + z20 = _mm512_xor_si512(z20, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z3 = _mm512_xor_si512(z3, z30); + z18 = _mm512_xor_si512(z18, z30); + z8 = _mm512_xor_si512(z8, z30); + z23 = _mm512_xor_si512(z23, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z1 = _mm512_xor_si512(z1, z30); + z16 = _mm512_xor_si512(z16, z30); + z6 = _mm512_xor_si512(z6, z30); + z21 = _mm512_xor_si512(z21, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z4 = _mm512_xor_si512(z4, z30); + z19 = _mm512_xor_si512(z19, z30); + z9 = _mm512_xor_si512(z9, z30); + z24 = _mm512_xor_si512(z24, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z2 = _mm512_xor_si512(z2, z30); + z17 = _mm512_xor_si512(z17, z30); + z7 = _mm512_xor_si512(z7, z30); + z22 = _mm512_xor_si512(z22, z30); + z12 = _mm512_xor_si512(z12, z30); + /* Rho - rotate each lane in place */ + z3 = _mm512_rol_epi64(z3, 1); + z1 = _mm512_rol_epi64(z1, 62); + z4 = _mm512_rol_epi64(z4, 28); + z2 = _mm512_rol_epi64(z2, 27); + z15 = _mm512_rol_epi64(z15, 36); + z18 = _mm512_rol_epi64(z18, 44); + z16 = _mm512_rol_epi64(z16, 6); + z19 = _mm512_rol_epi64(z19, 55); + z17 = _mm512_rol_epi64(z17, 20); + z5 = _mm512_rol_epi64(z5, 3); + z8 = _mm512_rol_epi64(z8, 10); + z6 = _mm512_rol_epi64(z6, 43); + z9 = _mm512_rol_epi64(z9, 25); + z7 = _mm512_rol_epi64(z7, 39); + z20 = _mm512_rol_epi64(z20, 41); + z23 = _mm512_rol_epi64(z23, 45); + z21 = _mm512_rol_epi64(z21, 15); + z24 = _mm512_rol_epi64(z24, 21); + z22 = _mm512_rol_epi64(z22, 8); + z10 = _mm512_rol_epi64(z10, 18); + z13 = _mm512_rol_epi64(z13, 2); + z11 = _mm512_rol_epi64(z11, 61); + z14 = _mm512_rol_epi64(z14, 56); + z12 = _mm512_rol_epi64(z12, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z18; + z0 = _mm512_ternarylogic_epi64(z0, z18, z6, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z6, z24, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z24, z12, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z4; + z31 = z17; + z4 = _mm512_ternarylogic_epi64(z4, z17, z5, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z5, z23, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z23, z11, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z3; + z31 = z16; + z3 = _mm512_ternarylogic_epi64(z3, z16, z9, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z9, z22, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z22, z10, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z2; + z31 = z15; + z2 = _mm512_ternarylogic_epi64(z2, z15, z8, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z8, z21, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z21, z14, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z1; + z31 = z19; + z1 = _mm512_ternarylogic_epi64(z1, z19, z7, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z7, z20, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z20, z13, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, + 1152))); + /* Round 19 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z4, z3, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z2, z1, 0x96); + z26 = z18; + z26 = _mm512_ternarylogic_epi64(z26, z17, z16, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z15, z19, 0x96); + z27 = z6; + z27 = _mm512_ternarylogic_epi64(z27, z5, z9, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z8, z7, 0x96); + z28 = z24; + z28 = _mm512_ternarylogic_epi64(z28, z23, z22, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z21, z20, 0x96); + z29 = z12; + z29 = _mm512_ternarylogic_epi64(z29, z11, z10, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z14, z13, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z4 = _mm512_xor_si512(z4, z30); + z3 = _mm512_xor_si512(z3, z30); + z2 = _mm512_xor_si512(z2, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z18 = _mm512_xor_si512(z18, z30); + z17 = _mm512_xor_si512(z17, z30); + z16 = _mm512_xor_si512(z16, z30); + z15 = _mm512_xor_si512(z15, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z6 = _mm512_xor_si512(z6, z30); + z5 = _mm512_xor_si512(z5, z30); + z9 = _mm512_xor_si512(z9, z30); + z8 = _mm512_xor_si512(z8, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z24 = _mm512_xor_si512(z24, z30); + z23 = _mm512_xor_si512(z23, z30); + z22 = _mm512_xor_si512(z22, z30); + z21 = _mm512_xor_si512(z21, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z12 = _mm512_xor_si512(z12, z30); + z11 = _mm512_xor_si512(z11, z30); + z10 = _mm512_xor_si512(z10, z30); + z14 = _mm512_xor_si512(z14, z30); + z13 = _mm512_xor_si512(z13, z30); + /* Rho - rotate each lane in place */ + z18 = _mm512_rol_epi64(z18, 1); + z6 = _mm512_rol_epi64(z6, 62); + z24 = _mm512_rol_epi64(z24, 28); + z12 = _mm512_rol_epi64(z12, 27); + z4 = _mm512_rol_epi64(z4, 36); + z17 = _mm512_rol_epi64(z17, 44); + z5 = _mm512_rol_epi64(z5, 6); + z23 = _mm512_rol_epi64(z23, 55); + z11 = _mm512_rol_epi64(z11, 20); + z3 = _mm512_rol_epi64(z3, 3); + z16 = _mm512_rol_epi64(z16, 10); + z9 = _mm512_rol_epi64(z9, 43); + z22 = _mm512_rol_epi64(z22, 25); + z10 = _mm512_rol_epi64(z10, 39); + z2 = _mm512_rol_epi64(z2, 41); + z15 = _mm512_rol_epi64(z15, 45); + z8 = _mm512_rol_epi64(z8, 15); + z21 = _mm512_rol_epi64(z21, 21); + z14 = _mm512_rol_epi64(z14, 8); + z1 = _mm512_rol_epi64(z1, 18); + z19 = _mm512_rol_epi64(z19, 2); + z7 = _mm512_rol_epi64(z7, 61); + z20 = _mm512_rol_epi64(z20, 56); + z13 = _mm512_rol_epi64(z13, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z17; + z0 = _mm512_ternarylogic_epi64(z0, z17, z9, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z9, z21, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z21, z13, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z24; + z31 = z11; + z24 = _mm512_ternarylogic_epi64(z24, z11, z3, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z3, z15, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z15, z7, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z18; + z31 = z5; + z18 = _mm512_ternarylogic_epi64(z18, z5, z22, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z22, z14, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z14, z1, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z12; + z31 = z4; + z12 = _mm512_ternarylogic_epi64(z12, z4, z16, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z16, z8, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z8, z20, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z6; + z31 = z23; + z6 = _mm512_ternarylogic_epi64(z6, z23, z10, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z10, z2, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z2, z19, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, + 1216))); + /* Round 20 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z24, z18, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z12, z6, 0x96); + z26 = z17; + z26 = _mm512_ternarylogic_epi64(z26, z11, z5, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z4, z23, 0x96); + z27 = z9; + z27 = _mm512_ternarylogic_epi64(z27, z3, z22, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z16, z10, 0x96); + z28 = z21; + z28 = _mm512_ternarylogic_epi64(z28, z15, z14, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z8, z2, 0x96); + z29 = z13; + z29 = _mm512_ternarylogic_epi64(z29, z7, z1, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z20, z19, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z24 = _mm512_xor_si512(z24, z30); + z18 = _mm512_xor_si512(z18, z30); + z12 = _mm512_xor_si512(z12, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z17 = _mm512_xor_si512(z17, z30); + z11 = _mm512_xor_si512(z11, z30); + z5 = _mm512_xor_si512(z5, z30); + z4 = _mm512_xor_si512(z4, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z9 = _mm512_xor_si512(z9, z30); + z3 = _mm512_xor_si512(z3, z30); + z22 = _mm512_xor_si512(z22, z30); + z16 = _mm512_xor_si512(z16, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z21 = _mm512_xor_si512(z21, z30); + z15 = _mm512_xor_si512(z15, z30); + z14 = _mm512_xor_si512(z14, z30); + z8 = _mm512_xor_si512(z8, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z13 = _mm512_xor_si512(z13, z30); + z7 = _mm512_xor_si512(z7, z30); + z1 = _mm512_xor_si512(z1, z30); + z20 = _mm512_xor_si512(z20, z30); + z19 = _mm512_xor_si512(z19, z30); + /* Rho - rotate each lane in place */ + z17 = _mm512_rol_epi64(z17, 1); + z9 = _mm512_rol_epi64(z9, 62); + z21 = _mm512_rol_epi64(z21, 28); + z13 = _mm512_rol_epi64(z13, 27); + z24 = _mm512_rol_epi64(z24, 36); + z11 = _mm512_rol_epi64(z11, 44); + z3 = _mm512_rol_epi64(z3, 6); + z15 = _mm512_rol_epi64(z15, 55); + z7 = _mm512_rol_epi64(z7, 20); + z18 = _mm512_rol_epi64(z18, 3); + z5 = _mm512_rol_epi64(z5, 10); + z22 = _mm512_rol_epi64(z22, 43); + z14 = _mm512_rol_epi64(z14, 25); + z1 = _mm512_rol_epi64(z1, 39); + z12 = _mm512_rol_epi64(z12, 41); + z4 = _mm512_rol_epi64(z4, 45); + z16 = _mm512_rol_epi64(z16, 15); + z8 = _mm512_rol_epi64(z8, 21); + z20 = _mm512_rol_epi64(z20, 8); + z6 = _mm512_rol_epi64(z6, 18); + z23 = _mm512_rol_epi64(z23, 2); + z10 = _mm512_rol_epi64(z10, 61); + z2 = _mm512_rol_epi64(z2, 56); + z19 = _mm512_rol_epi64(z19, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z11; + z0 = _mm512_ternarylogic_epi64(z0, z11, z22, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z22, z8, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z8, z19, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z21; + z31 = z7; + z21 = _mm512_ternarylogic_epi64(z21, z7, z18, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z18, z4, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z4, z10, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z17; + z31 = z3; + z17 = _mm512_ternarylogic_epi64(z17, z3, z14, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z14, z20, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z20, z6, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z13; + z31 = z24; + z13 = _mm512_ternarylogic_epi64(z13, z24, z5, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z5, z16, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z16, z2, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z9; + z31 = z15; + z9 = _mm512_ternarylogic_epi64(z9, z15, z1, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z1, z12, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z12, z23, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, + 1280))); + /* Round 21 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z21, z17, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z13, z9, 0x96); + z26 = z11; + z26 = _mm512_ternarylogic_epi64(z26, z7, z3, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z24, z15, 0x96); + z27 = z22; + z27 = _mm512_ternarylogic_epi64(z27, z18, z14, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z5, z1, 0x96); + z28 = z8; + z28 = _mm512_ternarylogic_epi64(z28, z4, z20, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z16, z12, 0x96); + z29 = z19; + z29 = _mm512_ternarylogic_epi64(z29, z10, z6, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z2, z23, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z21 = _mm512_xor_si512(z21, z30); + z17 = _mm512_xor_si512(z17, z30); + z13 = _mm512_xor_si512(z13, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z11 = _mm512_xor_si512(z11, z30); + z7 = _mm512_xor_si512(z7, z30); + z3 = _mm512_xor_si512(z3, z30); + z24 = _mm512_xor_si512(z24, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z22 = _mm512_xor_si512(z22, z30); + z18 = _mm512_xor_si512(z18, z30); + z14 = _mm512_xor_si512(z14, z30); + z5 = _mm512_xor_si512(z5, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z8 = _mm512_xor_si512(z8, z30); + z4 = _mm512_xor_si512(z4, z30); + z20 = _mm512_xor_si512(z20, z30); + z16 = _mm512_xor_si512(z16, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z19 = _mm512_xor_si512(z19, z30); + z10 = _mm512_xor_si512(z10, z30); + z6 = _mm512_xor_si512(z6, z30); + z2 = _mm512_xor_si512(z2, z30); + z23 = _mm512_xor_si512(z23, z30); + /* Rho - rotate each lane in place */ + z11 = _mm512_rol_epi64(z11, 1); + z22 = _mm512_rol_epi64(z22, 62); + z8 = _mm512_rol_epi64(z8, 28); + z19 = _mm512_rol_epi64(z19, 27); + z21 = _mm512_rol_epi64(z21, 36); + z7 = _mm512_rol_epi64(z7, 44); + z18 = _mm512_rol_epi64(z18, 6); + z4 = _mm512_rol_epi64(z4, 55); + z10 = _mm512_rol_epi64(z10, 20); + z17 = _mm512_rol_epi64(z17, 3); + z3 = _mm512_rol_epi64(z3, 10); + z14 = _mm512_rol_epi64(z14, 43); + z20 = _mm512_rol_epi64(z20, 25); + z6 = _mm512_rol_epi64(z6, 39); + z13 = _mm512_rol_epi64(z13, 41); + z24 = _mm512_rol_epi64(z24, 45); + z5 = _mm512_rol_epi64(z5, 15); + z16 = _mm512_rol_epi64(z16, 21); + z2 = _mm512_rol_epi64(z2, 8); + z9 = _mm512_rol_epi64(z9, 18); + z15 = _mm512_rol_epi64(z15, 2); + z1 = _mm512_rol_epi64(z1, 61); + z12 = _mm512_rol_epi64(z12, 56); + z23 = _mm512_rol_epi64(z23, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z7; + z0 = _mm512_ternarylogic_epi64(z0, z7, z14, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z14, z16, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z16, z23, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z8; + z31 = z10; + z8 = _mm512_ternarylogic_epi64(z8, z10, z17, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z17, z24, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z24, z1, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z11; + z31 = z18; + z11 = _mm512_ternarylogic_epi64(z11, z18, z20, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z20, z2, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z2, z9, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z19; + z31 = z21; + z19 = _mm512_ternarylogic_epi64(z19, z21, z3, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z3, z5, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z5, z12, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z22; + z31 = z4; + z22 = _mm512_ternarylogic_epi64(z22, z4, z6, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z6, z13, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z13, z15, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, + 1344))); + /* Round 22 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z8, z11, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z19, z22, 0x96); + z26 = z7; + z26 = _mm512_ternarylogic_epi64(z26, z10, z18, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z21, z4, 0x96); + z27 = z14; + z27 = _mm512_ternarylogic_epi64(z27, z17, z20, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z3, z6, 0x96); + z28 = z16; + z28 = _mm512_ternarylogic_epi64(z28, z24, z2, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z5, z13, 0x96); + z29 = z23; + z29 = _mm512_ternarylogic_epi64(z29, z1, z9, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z12, z15, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z8 = _mm512_xor_si512(z8, z30); + z11 = _mm512_xor_si512(z11, z30); + z19 = _mm512_xor_si512(z19, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z7 = _mm512_xor_si512(z7, z30); + z10 = _mm512_xor_si512(z10, z30); + z18 = _mm512_xor_si512(z18, z30); + z21 = _mm512_xor_si512(z21, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z14 = _mm512_xor_si512(z14, z30); + z17 = _mm512_xor_si512(z17, z30); + z20 = _mm512_xor_si512(z20, z30); + z3 = _mm512_xor_si512(z3, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z16 = _mm512_xor_si512(z16, z30); + z24 = _mm512_xor_si512(z24, z30); + z2 = _mm512_xor_si512(z2, z30); + z5 = _mm512_xor_si512(z5, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z23 = _mm512_xor_si512(z23, z30); + z1 = _mm512_xor_si512(z1, z30); + z9 = _mm512_xor_si512(z9, z30); + z12 = _mm512_xor_si512(z12, z30); + z15 = _mm512_xor_si512(z15, z30); + /* Rho - rotate each lane in place */ + z7 = _mm512_rol_epi64(z7, 1); + z14 = _mm512_rol_epi64(z14, 62); + z16 = _mm512_rol_epi64(z16, 28); + z23 = _mm512_rol_epi64(z23, 27); + z8 = _mm512_rol_epi64(z8, 36); + z10 = _mm512_rol_epi64(z10, 44); + z17 = _mm512_rol_epi64(z17, 6); + z24 = _mm512_rol_epi64(z24, 55); + z1 = _mm512_rol_epi64(z1, 20); + z11 = _mm512_rol_epi64(z11, 3); + z18 = _mm512_rol_epi64(z18, 10); + z20 = _mm512_rol_epi64(z20, 43); + z2 = _mm512_rol_epi64(z2, 25); + z9 = _mm512_rol_epi64(z9, 39); + z19 = _mm512_rol_epi64(z19, 41); + z21 = _mm512_rol_epi64(z21, 45); + z3 = _mm512_rol_epi64(z3, 15); + z5 = _mm512_rol_epi64(z5, 21); + z12 = _mm512_rol_epi64(z12, 8); + z22 = _mm512_rol_epi64(z22, 18); + z4 = _mm512_rol_epi64(z4, 2); + z6 = _mm512_rol_epi64(z6, 61); + z13 = _mm512_rol_epi64(z13, 56); + z15 = _mm512_rol_epi64(z15, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z10; + z0 = _mm512_ternarylogic_epi64(z0, z10, z20, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z20, z5, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z5, z15, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z16; + z31 = z1; + z16 = _mm512_ternarylogic_epi64(z16, z1, z11, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z11, z21, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z21, z6, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z7; + z31 = z17; + z7 = _mm512_ternarylogic_epi64(z7, z17, z2, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z2, z12, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z12, z22, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z23; + z31 = z8; + z23 = _mm512_ternarylogic_epi64(z23, z8, z18, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z18, z3, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z3, z13, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z14; + z31 = z24; + z14 = _mm512_ternarylogic_epi64(z14, z24, z9, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z9, z19, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z19, z4, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, + 1408))); + /* Round 23 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z16, z7, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z23, z14, 0x96); + z26 = z10; + z26 = _mm512_ternarylogic_epi64(z26, z1, z17, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z8, z24, 0x96); + z27 = z20; + z27 = _mm512_ternarylogic_epi64(z27, z11, z2, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z18, z9, 0x96); + z28 = z5; + z28 = _mm512_ternarylogic_epi64(z28, z21, z12, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z3, z19, 0x96); + z29 = z15; + z29 = _mm512_ternarylogic_epi64(z29, z6, z22, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z13, z4, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z16 = _mm512_xor_si512(z16, z30); + z7 = _mm512_xor_si512(z7, z30); + z23 = _mm512_xor_si512(z23, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z10 = _mm512_xor_si512(z10, z30); + z1 = _mm512_xor_si512(z1, z30); + z17 = _mm512_xor_si512(z17, z30); + z8 = _mm512_xor_si512(z8, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z20 = _mm512_xor_si512(z20, z30); + z11 = _mm512_xor_si512(z11, z30); + z2 = _mm512_xor_si512(z2, z30); + z18 = _mm512_xor_si512(z18, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z5 = _mm512_xor_si512(z5, z30); + z21 = _mm512_xor_si512(z21, z30); + z12 = _mm512_xor_si512(z12, z30); + z3 = _mm512_xor_si512(z3, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z15 = _mm512_xor_si512(z15, z30); + z6 = _mm512_xor_si512(z6, z30); + z22 = _mm512_xor_si512(z22, z30); + z13 = _mm512_xor_si512(z13, z30); + z4 = _mm512_xor_si512(z4, z30); + /* Rho - rotate each lane in place */ + z10 = _mm512_rol_epi64(z10, 1); + z20 = _mm512_rol_epi64(z20, 62); + z5 = _mm512_rol_epi64(z5, 28); + z15 = _mm512_rol_epi64(z15, 27); + z16 = _mm512_rol_epi64(z16, 36); + z1 = _mm512_rol_epi64(z1, 44); + z11 = _mm512_rol_epi64(z11, 6); + z21 = _mm512_rol_epi64(z21, 55); + z6 = _mm512_rol_epi64(z6, 20); + z7 = _mm512_rol_epi64(z7, 3); + z17 = _mm512_rol_epi64(z17, 10); + z2 = _mm512_rol_epi64(z2, 43); + z12 = _mm512_rol_epi64(z12, 25); + z22 = _mm512_rol_epi64(z22, 39); + z23 = _mm512_rol_epi64(z23, 41); + z8 = _mm512_rol_epi64(z8, 45); + z18 = _mm512_rol_epi64(z18, 15); + z3 = _mm512_rol_epi64(z3, 21); + z13 = _mm512_rol_epi64(z13, 8); + z14 = _mm512_rol_epi64(z14, 18); + z24 = _mm512_rol_epi64(z24, 2); + z9 = _mm512_rol_epi64(z9, 61); + z19 = _mm512_rol_epi64(z19, 56); + z4 = _mm512_rol_epi64(z4, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z1; + z0 = _mm512_ternarylogic_epi64(z0, z1, z2, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z2, z3, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z3, z4, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z5; + z31 = z6; + z5 = _mm512_ternarylogic_epi64(z5, z6, z7, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z7, z8, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z8, z9, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z10; + z31 = z11; + z10 = _mm512_ternarylogic_epi64(z10, z11, z12, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z12, z13, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z13, z14, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z15; + z31 = z16; + z15 = _mm512_ternarylogic_epi64(z15, z16, z17, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z17, z18, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z18, z19, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z20; + z31 = z21; + z20 = _mm512_ternarylogic_epi64(z20, z21, z22, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z22, z23, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z23, z24, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rax, + 1472))); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 1024), z16); + _mm512_storeu_si512((void*)WC_PW(rdi, 1088), z17); + _mm512_storeu_si512((void*)WC_PW(rdi, 1152), z18); + _mm512_storeu_si512((void*)WC_PW(rdi, 1216), z19); + _mm512_storeu_si512((void*)WC_PW(rdi, 1280), z20); + _mm512_storeu_si512((void*)WC_PW(rdi, 1344), z21); + _mm512_storeu_si512((void*)WC_PW(rdi, 1408), z22); + _mm512_storeu_si512((void*)WC_PW(rdi, 1472), z23); + _mm512_storeu_si512((void*)WC_PW(rdi, 1536), z24); + rbx = (word64)(rdi); + r15 = (word64)(0x15); +L_sha3_blocksx8_out_avx512_word: + rbp = (word64)(WC_L64(rbx, 0)); + WC_S64(rsi, 0) = (word64)(rbp); + rbp = (word64)(WC_L64(rbx, 8)); + WC_S64(rcx, 0) = (word64)(rbp); + rbp = (word64)(WC_L64(rbx, 16)); + WC_S64(r8, 0) = (word64)(rbp); + rbp = (word64)(WC_L64(rbx, 24)); + WC_S64(r9, 0) = (word64)(rbp); + rbp = (word64)(WC_L64(rbx, 32)); + WC_S64(r10, 0) = (word64)(rbp); + rbp = (word64)(WC_L64(rbx, 40)); + WC_S64(r11, 0) = (word64)(rbp); + rbp = (word64)(WC_L64(rbx, 48)); + WC_S64(r12, 0) = (word64)(rbp); + rbp = (word64)(WC_L64(rbx, 56)); + WC_S64(r13, 0) = (word64)(rbp); + rbx = (word64)(rbx + 0x40); + rsi = (word64)(rsi + 8); + rcx = (word64)(rcx + 8); + r8 = (word64)(r8 + 8); + r9 = (word64)(r9 + 8); + r10 = (word64)(r10 + 8); + r11 = (word64)(r11 + 8); + r12 = (word64)(r12 + 8); + r13 = (word64)(r13 + 8); + r14 = (word64)(r14 - 1); + if ((r14) == (0)) { + goto L_sha3_blocksx8_out_avx512_done; + } + r15 = (word64)(r15 - 1); + if ((r15) != (0)) { + goto L_sha3_blocksx8_out_avx512_word; + } + goto L_sha3_blocksx8_out_avx512_block; +L_sha3_blocksx8_out_avx512_done: + ; +} + +#endif /* WOLFSSL_HAVE_FRODOKEM */ +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void sha3_blocksx8_avx512(word64* s) +{ + word64 rdi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23, z24, z25, z26, z27, + z28, z29, z30, z31; + + rdi = (word64)(size_t)s; + + rdx = (word64)(L_sha3_x8_avx512_r); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z16 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1024)); + z17 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1088)); + z18 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1152)); + z19 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1216)); + z20 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1280)); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1344)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1408)); + z23 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1472)); + z24 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1536)); + /* Round 0 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z5, z10, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z15, z20, 0x96); + z26 = z1; + z26 = _mm512_ternarylogic_epi64(z26, z6, z11, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z16, z21, 0x96); + z27 = z2; + z27 = _mm512_ternarylogic_epi64(z27, z7, z12, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z17, z22, 0x96); + z28 = z3; + z28 = _mm512_ternarylogic_epi64(z28, z8, z13, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z18, z23, 0x96); + z29 = z4; + z29 = _mm512_ternarylogic_epi64(z29, z9, z14, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z19, z24, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z5 = _mm512_xor_si512(z5, z30); + z10 = _mm512_xor_si512(z10, z30); + z15 = _mm512_xor_si512(z15, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z1 = _mm512_xor_si512(z1, z30); + z6 = _mm512_xor_si512(z6, z30); + z11 = _mm512_xor_si512(z11, z30); + z16 = _mm512_xor_si512(z16, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z2 = _mm512_xor_si512(z2, z30); + z7 = _mm512_xor_si512(z7, z30); + z12 = _mm512_xor_si512(z12, z30); + z17 = _mm512_xor_si512(z17, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z3 = _mm512_xor_si512(z3, z30); + z8 = _mm512_xor_si512(z8, z30); + z13 = _mm512_xor_si512(z13, z30); + z18 = _mm512_xor_si512(z18, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z4 = _mm512_xor_si512(z4, z30); + z9 = _mm512_xor_si512(z9, z30); + z14 = _mm512_xor_si512(z14, z30); + z19 = _mm512_xor_si512(z19, z30); + z24 = _mm512_xor_si512(z24, z30); + /* Rho - rotate each lane in place */ + z1 = _mm512_rol_epi64(z1, 1); + z2 = _mm512_rol_epi64(z2, 62); + z3 = _mm512_rol_epi64(z3, 28); + z4 = _mm512_rol_epi64(z4, 27); + z5 = _mm512_rol_epi64(z5, 36); + z6 = _mm512_rol_epi64(z6, 44); + z7 = _mm512_rol_epi64(z7, 6); + z8 = _mm512_rol_epi64(z8, 55); + z9 = _mm512_rol_epi64(z9, 20); + z10 = _mm512_rol_epi64(z10, 3); + z11 = _mm512_rol_epi64(z11, 10); + z12 = _mm512_rol_epi64(z12, 43); + z13 = _mm512_rol_epi64(z13, 25); + z14 = _mm512_rol_epi64(z14, 39); + z15 = _mm512_rol_epi64(z15, 41); + z16 = _mm512_rol_epi64(z16, 45); + z17 = _mm512_rol_epi64(z17, 15); + z18 = _mm512_rol_epi64(z18, 21); + z19 = _mm512_rol_epi64(z19, 8); + z20 = _mm512_rol_epi64(z20, 18); + z21 = _mm512_rol_epi64(z21, 2); + z22 = _mm512_rol_epi64(z22, 61); + z23 = _mm512_rol_epi64(z23, 56); + z24 = _mm512_rol_epi64(z24, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z6; + z0 = _mm512_ternarylogic_epi64(z0, z6, z12, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z12, z18, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z18, z24, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z3; + z31 = z9; + z3 = _mm512_ternarylogic_epi64(z3, z9, z10, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z10, z16, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z16, z22, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z1; + z31 = z7; + z1 = _mm512_ternarylogic_epi64(z1, z7, z13, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z13, z19, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z19, z20, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z4; + z31 = z5; + z4 = _mm512_ternarylogic_epi64(z4, z5, z11, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z11, z17, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z17, z23, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z2; + z31 = z8; + z2 = _mm512_ternarylogic_epi64(z2, z8, z14, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z14, z15, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z15, z21, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 0))); + /* Round 1 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z3, z1, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z4, z2, 0x96); + z26 = z6; + z26 = _mm512_ternarylogic_epi64(z26, z9, z7, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z5, z8, 0x96); + z27 = z12; + z27 = _mm512_ternarylogic_epi64(z27, z10, z13, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z11, z14, 0x96); + z28 = z18; + z28 = _mm512_ternarylogic_epi64(z28, z16, z19, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z17, z15, 0x96); + z29 = z24; + z29 = _mm512_ternarylogic_epi64(z29, z22, z20, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z23, z21, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z3 = _mm512_xor_si512(z3, z30); + z1 = _mm512_xor_si512(z1, z30); + z4 = _mm512_xor_si512(z4, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z6 = _mm512_xor_si512(z6, z30); + z9 = _mm512_xor_si512(z9, z30); + z7 = _mm512_xor_si512(z7, z30); + z5 = _mm512_xor_si512(z5, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z12 = _mm512_xor_si512(z12, z30); + z10 = _mm512_xor_si512(z10, z30); + z13 = _mm512_xor_si512(z13, z30); + z11 = _mm512_xor_si512(z11, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z18 = _mm512_xor_si512(z18, z30); + z16 = _mm512_xor_si512(z16, z30); + z19 = _mm512_xor_si512(z19, z30); + z17 = _mm512_xor_si512(z17, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z24 = _mm512_xor_si512(z24, z30); + z22 = _mm512_xor_si512(z22, z30); + z20 = _mm512_xor_si512(z20, z30); + z23 = _mm512_xor_si512(z23, z30); + z21 = _mm512_xor_si512(z21, z30); + /* Rho - rotate each lane in place */ + z6 = _mm512_rol_epi64(z6, 1); + z12 = _mm512_rol_epi64(z12, 62); + z18 = _mm512_rol_epi64(z18, 28); + z24 = _mm512_rol_epi64(z24, 27); + z3 = _mm512_rol_epi64(z3, 36); + z9 = _mm512_rol_epi64(z9, 44); + z10 = _mm512_rol_epi64(z10, 6); + z16 = _mm512_rol_epi64(z16, 55); + z22 = _mm512_rol_epi64(z22, 20); + z1 = _mm512_rol_epi64(z1, 3); + z7 = _mm512_rol_epi64(z7, 10); + z13 = _mm512_rol_epi64(z13, 43); + z19 = _mm512_rol_epi64(z19, 25); + z20 = _mm512_rol_epi64(z20, 39); + z4 = _mm512_rol_epi64(z4, 41); + z5 = _mm512_rol_epi64(z5, 45); + z11 = _mm512_rol_epi64(z11, 15); + z17 = _mm512_rol_epi64(z17, 21); + z23 = _mm512_rol_epi64(z23, 8); + z2 = _mm512_rol_epi64(z2, 18); + z8 = _mm512_rol_epi64(z8, 2); + z14 = _mm512_rol_epi64(z14, 61); + z15 = _mm512_rol_epi64(z15, 56); + z21 = _mm512_rol_epi64(z21, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z9; + z0 = _mm512_ternarylogic_epi64(z0, z9, z13, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z13, z17, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z17, z21, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z18; + z31 = z22; + z18 = _mm512_ternarylogic_epi64(z18, z22, z1, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z1, z5, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z5, z14, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z6; + z31 = z10; + z6 = _mm512_ternarylogic_epi64(z6, z10, z19, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z19, z23, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z23, z2, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z24; + z31 = z3; + z24 = _mm512_ternarylogic_epi64(z24, z3, z7, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z7, z11, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z11, z15, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z12; + z31 = z16; + z12 = _mm512_ternarylogic_epi64(z12, z16, z20, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z20, z4, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z4, z8, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 64))); + /* Round 2 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z18, z6, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z24, z12, 0x96); + z26 = z9; + z26 = _mm512_ternarylogic_epi64(z26, z22, z10, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z3, z16, 0x96); + z27 = z13; + z27 = _mm512_ternarylogic_epi64(z27, z1, z19, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z7, z20, 0x96); + z28 = z17; + z28 = _mm512_ternarylogic_epi64(z28, z5, z23, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z11, z4, 0x96); + z29 = z21; + z29 = _mm512_ternarylogic_epi64(z29, z14, z2, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z15, z8, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z18 = _mm512_xor_si512(z18, z30); + z6 = _mm512_xor_si512(z6, z30); + z24 = _mm512_xor_si512(z24, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z9 = _mm512_xor_si512(z9, z30); + z22 = _mm512_xor_si512(z22, z30); + z10 = _mm512_xor_si512(z10, z30); + z3 = _mm512_xor_si512(z3, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z13 = _mm512_xor_si512(z13, z30); + z1 = _mm512_xor_si512(z1, z30); + z19 = _mm512_xor_si512(z19, z30); + z7 = _mm512_xor_si512(z7, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z17 = _mm512_xor_si512(z17, z30); + z5 = _mm512_xor_si512(z5, z30); + z23 = _mm512_xor_si512(z23, z30); + z11 = _mm512_xor_si512(z11, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z21 = _mm512_xor_si512(z21, z30); + z14 = _mm512_xor_si512(z14, z30); + z2 = _mm512_xor_si512(z2, z30); + z15 = _mm512_xor_si512(z15, z30); + z8 = _mm512_xor_si512(z8, z30); + /* Rho - rotate each lane in place */ + z9 = _mm512_rol_epi64(z9, 1); + z13 = _mm512_rol_epi64(z13, 62); + z17 = _mm512_rol_epi64(z17, 28); + z21 = _mm512_rol_epi64(z21, 27); + z18 = _mm512_rol_epi64(z18, 36); + z22 = _mm512_rol_epi64(z22, 44); + z1 = _mm512_rol_epi64(z1, 6); + z5 = _mm512_rol_epi64(z5, 55); + z14 = _mm512_rol_epi64(z14, 20); + z6 = _mm512_rol_epi64(z6, 3); + z10 = _mm512_rol_epi64(z10, 10); + z19 = _mm512_rol_epi64(z19, 43); + z23 = _mm512_rol_epi64(z23, 25); + z2 = _mm512_rol_epi64(z2, 39); + z24 = _mm512_rol_epi64(z24, 41); + z3 = _mm512_rol_epi64(z3, 45); + z7 = _mm512_rol_epi64(z7, 15); + z11 = _mm512_rol_epi64(z11, 21); + z15 = _mm512_rol_epi64(z15, 8); + z12 = _mm512_rol_epi64(z12, 18); + z16 = _mm512_rol_epi64(z16, 2); + z20 = _mm512_rol_epi64(z20, 61); + z4 = _mm512_rol_epi64(z4, 56); + z8 = _mm512_rol_epi64(z8, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z22; + z0 = _mm512_ternarylogic_epi64(z0, z22, z19, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z19, z11, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z11, z8, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z17; + z31 = z14; + z17 = _mm512_ternarylogic_epi64(z17, z14, z6, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z6, z3, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z3, z20, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z9; + z31 = z1; + z9 = _mm512_ternarylogic_epi64(z9, z1, z23, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z23, z15, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z15, z12, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z21; + z31 = z18; + z21 = _mm512_ternarylogic_epi64(z21, z18, z10, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z10, z7, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z7, z4, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z13; + z31 = z5; + z13 = _mm512_ternarylogic_epi64(z13, z5, z2, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z2, z24, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z24, z16, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 128))); + /* Round 3 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z17, z9, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z21, z13, 0x96); + z26 = z22; + z26 = _mm512_ternarylogic_epi64(z26, z14, z1, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z18, z5, 0x96); + z27 = z19; + z27 = _mm512_ternarylogic_epi64(z27, z6, z23, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z10, z2, 0x96); + z28 = z11; + z28 = _mm512_ternarylogic_epi64(z28, z3, z15, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z7, z24, 0x96); + z29 = z8; + z29 = _mm512_ternarylogic_epi64(z29, z20, z12, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z4, z16, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z17 = _mm512_xor_si512(z17, z30); + z9 = _mm512_xor_si512(z9, z30); + z21 = _mm512_xor_si512(z21, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z22 = _mm512_xor_si512(z22, z30); + z14 = _mm512_xor_si512(z14, z30); + z1 = _mm512_xor_si512(z1, z30); + z18 = _mm512_xor_si512(z18, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z19 = _mm512_xor_si512(z19, z30); + z6 = _mm512_xor_si512(z6, z30); + z23 = _mm512_xor_si512(z23, z30); + z10 = _mm512_xor_si512(z10, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z11 = _mm512_xor_si512(z11, z30); + z3 = _mm512_xor_si512(z3, z30); + z15 = _mm512_xor_si512(z15, z30); + z7 = _mm512_xor_si512(z7, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z8 = _mm512_xor_si512(z8, z30); + z20 = _mm512_xor_si512(z20, z30); + z12 = _mm512_xor_si512(z12, z30); + z4 = _mm512_xor_si512(z4, z30); + z16 = _mm512_xor_si512(z16, z30); + /* Rho - rotate each lane in place */ + z22 = _mm512_rol_epi64(z22, 1); + z19 = _mm512_rol_epi64(z19, 62); + z11 = _mm512_rol_epi64(z11, 28); + z8 = _mm512_rol_epi64(z8, 27); + z17 = _mm512_rol_epi64(z17, 36); + z14 = _mm512_rol_epi64(z14, 44); + z6 = _mm512_rol_epi64(z6, 6); + z3 = _mm512_rol_epi64(z3, 55); + z20 = _mm512_rol_epi64(z20, 20); + z9 = _mm512_rol_epi64(z9, 3); + z1 = _mm512_rol_epi64(z1, 10); + z23 = _mm512_rol_epi64(z23, 43); + z15 = _mm512_rol_epi64(z15, 25); + z12 = _mm512_rol_epi64(z12, 39); + z21 = _mm512_rol_epi64(z21, 41); + z18 = _mm512_rol_epi64(z18, 45); + z10 = _mm512_rol_epi64(z10, 15); + z7 = _mm512_rol_epi64(z7, 21); + z4 = _mm512_rol_epi64(z4, 8); + z13 = _mm512_rol_epi64(z13, 18); + z5 = _mm512_rol_epi64(z5, 2); + z2 = _mm512_rol_epi64(z2, 61); + z24 = _mm512_rol_epi64(z24, 56); + z16 = _mm512_rol_epi64(z16, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z14; + z0 = _mm512_ternarylogic_epi64(z0, z14, z23, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z23, z7, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z7, z16, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z11; + z31 = z20; + z11 = _mm512_ternarylogic_epi64(z11, z20, z9, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z9, z18, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z18, z2, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z22; + z31 = z6; + z22 = _mm512_ternarylogic_epi64(z22, z6, z15, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z15, z4, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z4, z13, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z8; + z31 = z17; + z8 = _mm512_ternarylogic_epi64(z8, z17, z1, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z1, z10, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z10, z24, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z19; + z31 = z3; + z19 = _mm512_ternarylogic_epi64(z19, z3, z12, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z12, z21, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z21, z5, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 192))); + /* Round 4 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z11, z22, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z8, z19, 0x96); + z26 = z14; + z26 = _mm512_ternarylogic_epi64(z26, z20, z6, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z17, z3, 0x96); + z27 = z23; + z27 = _mm512_ternarylogic_epi64(z27, z9, z15, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z1, z12, 0x96); + z28 = z7; + z28 = _mm512_ternarylogic_epi64(z28, z18, z4, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z10, z21, 0x96); + z29 = z16; + z29 = _mm512_ternarylogic_epi64(z29, z2, z13, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z24, z5, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z11 = _mm512_xor_si512(z11, z30); + z22 = _mm512_xor_si512(z22, z30); + z8 = _mm512_xor_si512(z8, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z14 = _mm512_xor_si512(z14, z30); + z20 = _mm512_xor_si512(z20, z30); + z6 = _mm512_xor_si512(z6, z30); + z17 = _mm512_xor_si512(z17, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z23 = _mm512_xor_si512(z23, z30); + z9 = _mm512_xor_si512(z9, z30); + z15 = _mm512_xor_si512(z15, z30); + z1 = _mm512_xor_si512(z1, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z7 = _mm512_xor_si512(z7, z30); + z18 = _mm512_xor_si512(z18, z30); + z4 = _mm512_xor_si512(z4, z30); + z10 = _mm512_xor_si512(z10, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z16 = _mm512_xor_si512(z16, z30); + z2 = _mm512_xor_si512(z2, z30); + z13 = _mm512_xor_si512(z13, z30); + z24 = _mm512_xor_si512(z24, z30); + z5 = _mm512_xor_si512(z5, z30); + /* Rho - rotate each lane in place */ + z14 = _mm512_rol_epi64(z14, 1); + z23 = _mm512_rol_epi64(z23, 62); + z7 = _mm512_rol_epi64(z7, 28); + z16 = _mm512_rol_epi64(z16, 27); + z11 = _mm512_rol_epi64(z11, 36); + z20 = _mm512_rol_epi64(z20, 44); + z9 = _mm512_rol_epi64(z9, 6); + z18 = _mm512_rol_epi64(z18, 55); + z2 = _mm512_rol_epi64(z2, 20); + z22 = _mm512_rol_epi64(z22, 3); + z6 = _mm512_rol_epi64(z6, 10); + z15 = _mm512_rol_epi64(z15, 43); + z4 = _mm512_rol_epi64(z4, 25); + z13 = _mm512_rol_epi64(z13, 39); + z8 = _mm512_rol_epi64(z8, 41); + z17 = _mm512_rol_epi64(z17, 45); + z1 = _mm512_rol_epi64(z1, 15); + z10 = _mm512_rol_epi64(z10, 21); + z24 = _mm512_rol_epi64(z24, 8); + z19 = _mm512_rol_epi64(z19, 18); + z3 = _mm512_rol_epi64(z3, 2); + z12 = _mm512_rol_epi64(z12, 61); + z21 = _mm512_rol_epi64(z21, 56); + z5 = _mm512_rol_epi64(z5, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z20; + z0 = _mm512_ternarylogic_epi64(z0, z20, z15, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z15, z10, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z10, z5, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z7; + z31 = z2; + z7 = _mm512_ternarylogic_epi64(z7, z2, z22, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z22, z17, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z17, z12, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z14; + z31 = z9; + z14 = _mm512_ternarylogic_epi64(z14, z9, z4, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z4, z24, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z24, z19, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z16; + z31 = z11; + z16 = _mm512_ternarylogic_epi64(z16, z11, z6, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z6, z1, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z1, z21, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z23; + z31 = z18; + z23 = _mm512_ternarylogic_epi64(z23, z18, z13, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z13, z8, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z8, z3, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 256))); + /* Round 5 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z7, z14, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z16, z23, 0x96); + z26 = z20; + z26 = _mm512_ternarylogic_epi64(z26, z2, z9, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z11, z18, 0x96); + z27 = z15; + z27 = _mm512_ternarylogic_epi64(z27, z22, z4, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z6, z13, 0x96); + z28 = z10; + z28 = _mm512_ternarylogic_epi64(z28, z17, z24, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z1, z8, 0x96); + z29 = z5; + z29 = _mm512_ternarylogic_epi64(z29, z12, z19, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z21, z3, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z7 = _mm512_xor_si512(z7, z30); + z14 = _mm512_xor_si512(z14, z30); + z16 = _mm512_xor_si512(z16, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z20 = _mm512_xor_si512(z20, z30); + z2 = _mm512_xor_si512(z2, z30); + z9 = _mm512_xor_si512(z9, z30); + z11 = _mm512_xor_si512(z11, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z15 = _mm512_xor_si512(z15, z30); + z22 = _mm512_xor_si512(z22, z30); + z4 = _mm512_xor_si512(z4, z30); + z6 = _mm512_xor_si512(z6, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z10 = _mm512_xor_si512(z10, z30); + z17 = _mm512_xor_si512(z17, z30); + z24 = _mm512_xor_si512(z24, z30); + z1 = _mm512_xor_si512(z1, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z5 = _mm512_xor_si512(z5, z30); + z12 = _mm512_xor_si512(z12, z30); + z19 = _mm512_xor_si512(z19, z30); + z21 = _mm512_xor_si512(z21, z30); + z3 = _mm512_xor_si512(z3, z30); + /* Rho - rotate each lane in place */ + z20 = _mm512_rol_epi64(z20, 1); + z15 = _mm512_rol_epi64(z15, 62); + z10 = _mm512_rol_epi64(z10, 28); + z5 = _mm512_rol_epi64(z5, 27); + z7 = _mm512_rol_epi64(z7, 36); + z2 = _mm512_rol_epi64(z2, 44); + z22 = _mm512_rol_epi64(z22, 6); + z17 = _mm512_rol_epi64(z17, 55); + z12 = _mm512_rol_epi64(z12, 20); + z14 = _mm512_rol_epi64(z14, 3); + z9 = _mm512_rol_epi64(z9, 10); + z4 = _mm512_rol_epi64(z4, 43); + z24 = _mm512_rol_epi64(z24, 25); + z19 = _mm512_rol_epi64(z19, 39); + z16 = _mm512_rol_epi64(z16, 41); + z11 = _mm512_rol_epi64(z11, 45); + z6 = _mm512_rol_epi64(z6, 15); + z1 = _mm512_rol_epi64(z1, 21); + z21 = _mm512_rol_epi64(z21, 8); + z23 = _mm512_rol_epi64(z23, 18); + z18 = _mm512_rol_epi64(z18, 2); + z13 = _mm512_rol_epi64(z13, 61); + z8 = _mm512_rol_epi64(z8, 56); + z3 = _mm512_rol_epi64(z3, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z2; + z0 = _mm512_ternarylogic_epi64(z0, z2, z4, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z4, z1, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z1, z3, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z10; + z31 = z12; + z10 = _mm512_ternarylogic_epi64(z10, z12, z14, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z14, z11, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z11, z13, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z20; + z31 = z22; + z20 = _mm512_ternarylogic_epi64(z20, z22, z24, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z24, z21, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z21, z23, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z5; + z31 = z7; + z5 = _mm512_ternarylogic_epi64(z5, z7, z9, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z9, z6, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z6, z8, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z15; + z31 = z17; + z15 = _mm512_ternarylogic_epi64(z15, z17, z19, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z19, z16, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z16, z18, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 320))); + /* Round 6 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z10, z20, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z5, z15, 0x96); + z26 = z2; + z26 = _mm512_ternarylogic_epi64(z26, z12, z22, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z7, z17, 0x96); + z27 = z4; + z27 = _mm512_ternarylogic_epi64(z27, z14, z24, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z9, z19, 0x96); + z28 = z1; + z28 = _mm512_ternarylogic_epi64(z28, z11, z21, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z6, z16, 0x96); + z29 = z3; + z29 = _mm512_ternarylogic_epi64(z29, z13, z23, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z8, z18, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z10 = _mm512_xor_si512(z10, z30); + z20 = _mm512_xor_si512(z20, z30); + z5 = _mm512_xor_si512(z5, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z2 = _mm512_xor_si512(z2, z30); + z12 = _mm512_xor_si512(z12, z30); + z22 = _mm512_xor_si512(z22, z30); + z7 = _mm512_xor_si512(z7, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z4 = _mm512_xor_si512(z4, z30); + z14 = _mm512_xor_si512(z14, z30); + z24 = _mm512_xor_si512(z24, z30); + z9 = _mm512_xor_si512(z9, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z1 = _mm512_xor_si512(z1, z30); + z11 = _mm512_xor_si512(z11, z30); + z21 = _mm512_xor_si512(z21, z30); + z6 = _mm512_xor_si512(z6, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z3 = _mm512_xor_si512(z3, z30); + z13 = _mm512_xor_si512(z13, z30); + z23 = _mm512_xor_si512(z23, z30); + z8 = _mm512_xor_si512(z8, z30); + z18 = _mm512_xor_si512(z18, z30); + /* Rho - rotate each lane in place */ + z2 = _mm512_rol_epi64(z2, 1); + z4 = _mm512_rol_epi64(z4, 62); + z1 = _mm512_rol_epi64(z1, 28); + z3 = _mm512_rol_epi64(z3, 27); + z10 = _mm512_rol_epi64(z10, 36); + z12 = _mm512_rol_epi64(z12, 44); + z14 = _mm512_rol_epi64(z14, 6); + z11 = _mm512_rol_epi64(z11, 55); + z13 = _mm512_rol_epi64(z13, 20); + z20 = _mm512_rol_epi64(z20, 3); + z22 = _mm512_rol_epi64(z22, 10); + z24 = _mm512_rol_epi64(z24, 43); + z21 = _mm512_rol_epi64(z21, 25); + z23 = _mm512_rol_epi64(z23, 39); + z5 = _mm512_rol_epi64(z5, 41); + z7 = _mm512_rol_epi64(z7, 45); + z9 = _mm512_rol_epi64(z9, 15); + z6 = _mm512_rol_epi64(z6, 21); + z8 = _mm512_rol_epi64(z8, 8); + z15 = _mm512_rol_epi64(z15, 18); + z17 = _mm512_rol_epi64(z17, 2); + z19 = _mm512_rol_epi64(z19, 61); + z16 = _mm512_rol_epi64(z16, 56); + z18 = _mm512_rol_epi64(z18, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z12; + z0 = _mm512_ternarylogic_epi64(z0, z12, z24, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z24, z6, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z6, z18, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z1; + z31 = z13; + z1 = _mm512_ternarylogic_epi64(z1, z13, z20, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z20, z7, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z7, z19, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z2; + z31 = z14; + z2 = _mm512_ternarylogic_epi64(z2, z14, z21, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z21, z8, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z8, z15, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z3; + z31 = z10; + z3 = _mm512_ternarylogic_epi64(z3, z10, z22, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z22, z9, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z9, z16, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z4; + z31 = z11; + z4 = _mm512_ternarylogic_epi64(z4, z11, z23, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z23, z5, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z5, z17, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 384))); + /* Round 7 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z1, z2, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z3, z4, 0x96); + z26 = z12; + z26 = _mm512_ternarylogic_epi64(z26, z13, z14, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z10, z11, 0x96); + z27 = z24; + z27 = _mm512_ternarylogic_epi64(z27, z20, z21, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z22, z23, 0x96); + z28 = z6; + z28 = _mm512_ternarylogic_epi64(z28, z7, z8, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z9, z5, 0x96); + z29 = z18; + z29 = _mm512_ternarylogic_epi64(z29, z19, z15, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z16, z17, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z1 = _mm512_xor_si512(z1, z30); + z2 = _mm512_xor_si512(z2, z30); + z3 = _mm512_xor_si512(z3, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z12 = _mm512_xor_si512(z12, z30); + z13 = _mm512_xor_si512(z13, z30); + z14 = _mm512_xor_si512(z14, z30); + z10 = _mm512_xor_si512(z10, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z24 = _mm512_xor_si512(z24, z30); + z20 = _mm512_xor_si512(z20, z30); + z21 = _mm512_xor_si512(z21, z30); + z22 = _mm512_xor_si512(z22, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z6 = _mm512_xor_si512(z6, z30); + z7 = _mm512_xor_si512(z7, z30); + z8 = _mm512_xor_si512(z8, z30); + z9 = _mm512_xor_si512(z9, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z18 = _mm512_xor_si512(z18, z30); + z19 = _mm512_xor_si512(z19, z30); + z15 = _mm512_xor_si512(z15, z30); + z16 = _mm512_xor_si512(z16, z30); + z17 = _mm512_xor_si512(z17, z30); + /* Rho - rotate each lane in place */ + z12 = _mm512_rol_epi64(z12, 1); + z24 = _mm512_rol_epi64(z24, 62); + z6 = _mm512_rol_epi64(z6, 28); + z18 = _mm512_rol_epi64(z18, 27); + z1 = _mm512_rol_epi64(z1, 36); + z13 = _mm512_rol_epi64(z13, 44); + z20 = _mm512_rol_epi64(z20, 6); + z7 = _mm512_rol_epi64(z7, 55); + z19 = _mm512_rol_epi64(z19, 20); + z2 = _mm512_rol_epi64(z2, 3); + z14 = _mm512_rol_epi64(z14, 10); + z21 = _mm512_rol_epi64(z21, 43); + z8 = _mm512_rol_epi64(z8, 25); + z15 = _mm512_rol_epi64(z15, 39); + z3 = _mm512_rol_epi64(z3, 41); + z10 = _mm512_rol_epi64(z10, 45); + z22 = _mm512_rol_epi64(z22, 15); + z9 = _mm512_rol_epi64(z9, 21); + z16 = _mm512_rol_epi64(z16, 8); + z4 = _mm512_rol_epi64(z4, 18); + z11 = _mm512_rol_epi64(z11, 2); + z23 = _mm512_rol_epi64(z23, 61); + z5 = _mm512_rol_epi64(z5, 56); + z17 = _mm512_rol_epi64(z17, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z13; + z0 = _mm512_ternarylogic_epi64(z0, z13, z21, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z21, z9, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z9, z17, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z6; + z31 = z19; + z6 = _mm512_ternarylogic_epi64(z6, z19, z2, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z2, z10, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z10, z23, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z12; + z31 = z20; + z12 = _mm512_ternarylogic_epi64(z12, z20, z8, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z8, z16, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z16, z4, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z18; + z31 = z1; + z18 = _mm512_ternarylogic_epi64(z18, z1, z14, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z14, z22, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z22, z5, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z24; + z31 = z7; + z24 = _mm512_ternarylogic_epi64(z24, z7, z15, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z15, z3, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z3, z11, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 448))); + /* Round 8 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z6, z12, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z18, z24, 0x96); + z26 = z13; + z26 = _mm512_ternarylogic_epi64(z26, z19, z20, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z1, z7, 0x96); + z27 = z21; + z27 = _mm512_ternarylogic_epi64(z27, z2, z8, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z14, z15, 0x96); + z28 = z9; + z28 = _mm512_ternarylogic_epi64(z28, z10, z16, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z22, z3, 0x96); + z29 = z17; + z29 = _mm512_ternarylogic_epi64(z29, z23, z4, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z5, z11, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z6 = _mm512_xor_si512(z6, z30); + z12 = _mm512_xor_si512(z12, z30); + z18 = _mm512_xor_si512(z18, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z13 = _mm512_xor_si512(z13, z30); + z19 = _mm512_xor_si512(z19, z30); + z20 = _mm512_xor_si512(z20, z30); + z1 = _mm512_xor_si512(z1, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z21 = _mm512_xor_si512(z21, z30); + z2 = _mm512_xor_si512(z2, z30); + z8 = _mm512_xor_si512(z8, z30); + z14 = _mm512_xor_si512(z14, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z9 = _mm512_xor_si512(z9, z30); + z10 = _mm512_xor_si512(z10, z30); + z16 = _mm512_xor_si512(z16, z30); + z22 = _mm512_xor_si512(z22, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z17 = _mm512_xor_si512(z17, z30); + z23 = _mm512_xor_si512(z23, z30); + z4 = _mm512_xor_si512(z4, z30); + z5 = _mm512_xor_si512(z5, z30); + z11 = _mm512_xor_si512(z11, z30); + /* Rho - rotate each lane in place */ + z13 = _mm512_rol_epi64(z13, 1); + z21 = _mm512_rol_epi64(z21, 62); + z9 = _mm512_rol_epi64(z9, 28); + z17 = _mm512_rol_epi64(z17, 27); + z6 = _mm512_rol_epi64(z6, 36); + z19 = _mm512_rol_epi64(z19, 44); + z2 = _mm512_rol_epi64(z2, 6); + z10 = _mm512_rol_epi64(z10, 55); + z23 = _mm512_rol_epi64(z23, 20); + z12 = _mm512_rol_epi64(z12, 3); + z20 = _mm512_rol_epi64(z20, 10); + z8 = _mm512_rol_epi64(z8, 43); + z16 = _mm512_rol_epi64(z16, 25); + z4 = _mm512_rol_epi64(z4, 39); + z18 = _mm512_rol_epi64(z18, 41); + z1 = _mm512_rol_epi64(z1, 45); + z14 = _mm512_rol_epi64(z14, 15); + z22 = _mm512_rol_epi64(z22, 21); + z5 = _mm512_rol_epi64(z5, 8); + z24 = _mm512_rol_epi64(z24, 18); + z7 = _mm512_rol_epi64(z7, 2); + z15 = _mm512_rol_epi64(z15, 61); + z3 = _mm512_rol_epi64(z3, 56); + z11 = _mm512_rol_epi64(z11, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z19; + z0 = _mm512_ternarylogic_epi64(z0, z19, z8, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z8, z22, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z22, z11, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z9; + z31 = z23; + z9 = _mm512_ternarylogic_epi64(z9, z23, z12, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z12, z1, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z1, z15, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z13; + z31 = z2; + z13 = _mm512_ternarylogic_epi64(z13, z2, z16, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z16, z5, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z5, z24, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z17; + z31 = z6; + z17 = _mm512_ternarylogic_epi64(z17, z6, z20, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z20, z14, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z14, z3, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z21; + z31 = z10; + z21 = _mm512_ternarylogic_epi64(z21, z10, z4, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z4, z18, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z18, z7, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 512))); + /* Round 9 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z9, z13, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z17, z21, 0x96); + z26 = z19; + z26 = _mm512_ternarylogic_epi64(z26, z23, z2, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z6, z10, 0x96); + z27 = z8; + z27 = _mm512_ternarylogic_epi64(z27, z12, z16, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z20, z4, 0x96); + z28 = z22; + z28 = _mm512_ternarylogic_epi64(z28, z1, z5, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z14, z18, 0x96); + z29 = z11; + z29 = _mm512_ternarylogic_epi64(z29, z15, z24, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z3, z7, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z9 = _mm512_xor_si512(z9, z30); + z13 = _mm512_xor_si512(z13, z30); + z17 = _mm512_xor_si512(z17, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z19 = _mm512_xor_si512(z19, z30); + z23 = _mm512_xor_si512(z23, z30); + z2 = _mm512_xor_si512(z2, z30); + z6 = _mm512_xor_si512(z6, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z8 = _mm512_xor_si512(z8, z30); + z12 = _mm512_xor_si512(z12, z30); + z16 = _mm512_xor_si512(z16, z30); + z20 = _mm512_xor_si512(z20, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z22 = _mm512_xor_si512(z22, z30); + z1 = _mm512_xor_si512(z1, z30); + z5 = _mm512_xor_si512(z5, z30); + z14 = _mm512_xor_si512(z14, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z11 = _mm512_xor_si512(z11, z30); + z15 = _mm512_xor_si512(z15, z30); + z24 = _mm512_xor_si512(z24, z30); + z3 = _mm512_xor_si512(z3, z30); + z7 = _mm512_xor_si512(z7, z30); + /* Rho - rotate each lane in place */ + z19 = _mm512_rol_epi64(z19, 1); + z8 = _mm512_rol_epi64(z8, 62); + z22 = _mm512_rol_epi64(z22, 28); + z11 = _mm512_rol_epi64(z11, 27); + z9 = _mm512_rol_epi64(z9, 36); + z23 = _mm512_rol_epi64(z23, 44); + z12 = _mm512_rol_epi64(z12, 6); + z1 = _mm512_rol_epi64(z1, 55); + z15 = _mm512_rol_epi64(z15, 20); + z13 = _mm512_rol_epi64(z13, 3); + z2 = _mm512_rol_epi64(z2, 10); + z16 = _mm512_rol_epi64(z16, 43); + z5 = _mm512_rol_epi64(z5, 25); + z24 = _mm512_rol_epi64(z24, 39); + z17 = _mm512_rol_epi64(z17, 41); + z6 = _mm512_rol_epi64(z6, 45); + z20 = _mm512_rol_epi64(z20, 15); + z14 = _mm512_rol_epi64(z14, 21); + z3 = _mm512_rol_epi64(z3, 8); + z21 = _mm512_rol_epi64(z21, 18); + z10 = _mm512_rol_epi64(z10, 2); + z4 = _mm512_rol_epi64(z4, 61); + z18 = _mm512_rol_epi64(z18, 56); + z7 = _mm512_rol_epi64(z7, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z23; + z0 = _mm512_ternarylogic_epi64(z0, z23, z16, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z16, z14, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z14, z7, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z22; + z31 = z15; + z22 = _mm512_ternarylogic_epi64(z22, z15, z13, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z13, z6, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z6, z4, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z19; + z31 = z12; + z19 = _mm512_ternarylogic_epi64(z19, z12, z5, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z5, z3, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z3, z21, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z11; + z31 = z9; + z11 = _mm512_ternarylogic_epi64(z11, z9, z2, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z2, z20, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z20, z18, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z8; + z31 = z1; + z8 = _mm512_ternarylogic_epi64(z8, z1, z24, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z24, z17, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z17, z10, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 576))); + /* Round 10 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z22, z19, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z11, z8, 0x96); + z26 = z23; + z26 = _mm512_ternarylogic_epi64(z26, z15, z12, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z9, z1, 0x96); + z27 = z16; + z27 = _mm512_ternarylogic_epi64(z27, z13, z5, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z2, z24, 0x96); + z28 = z14; + z28 = _mm512_ternarylogic_epi64(z28, z6, z3, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z20, z17, 0x96); + z29 = z7; + z29 = _mm512_ternarylogic_epi64(z29, z4, z21, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z18, z10, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z22 = _mm512_xor_si512(z22, z30); + z19 = _mm512_xor_si512(z19, z30); + z11 = _mm512_xor_si512(z11, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z23 = _mm512_xor_si512(z23, z30); + z15 = _mm512_xor_si512(z15, z30); + z12 = _mm512_xor_si512(z12, z30); + z9 = _mm512_xor_si512(z9, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z16 = _mm512_xor_si512(z16, z30); + z13 = _mm512_xor_si512(z13, z30); + z5 = _mm512_xor_si512(z5, z30); + z2 = _mm512_xor_si512(z2, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z14 = _mm512_xor_si512(z14, z30); + z6 = _mm512_xor_si512(z6, z30); + z3 = _mm512_xor_si512(z3, z30); + z20 = _mm512_xor_si512(z20, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z7 = _mm512_xor_si512(z7, z30); + z4 = _mm512_xor_si512(z4, z30); + z21 = _mm512_xor_si512(z21, z30); + z18 = _mm512_xor_si512(z18, z30); + z10 = _mm512_xor_si512(z10, z30); + /* Rho - rotate each lane in place */ + z23 = _mm512_rol_epi64(z23, 1); + z16 = _mm512_rol_epi64(z16, 62); + z14 = _mm512_rol_epi64(z14, 28); + z7 = _mm512_rol_epi64(z7, 27); + z22 = _mm512_rol_epi64(z22, 36); + z15 = _mm512_rol_epi64(z15, 44); + z13 = _mm512_rol_epi64(z13, 6); + z6 = _mm512_rol_epi64(z6, 55); + z4 = _mm512_rol_epi64(z4, 20); + z19 = _mm512_rol_epi64(z19, 3); + z12 = _mm512_rol_epi64(z12, 10); + z5 = _mm512_rol_epi64(z5, 43); + z3 = _mm512_rol_epi64(z3, 25); + z21 = _mm512_rol_epi64(z21, 39); + z11 = _mm512_rol_epi64(z11, 41); + z9 = _mm512_rol_epi64(z9, 45); + z2 = _mm512_rol_epi64(z2, 15); + z20 = _mm512_rol_epi64(z20, 21); + z18 = _mm512_rol_epi64(z18, 8); + z8 = _mm512_rol_epi64(z8, 18); + z1 = _mm512_rol_epi64(z1, 2); + z24 = _mm512_rol_epi64(z24, 61); + z17 = _mm512_rol_epi64(z17, 56); + z10 = _mm512_rol_epi64(z10, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z15; + z0 = _mm512_ternarylogic_epi64(z0, z15, z5, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z5, z20, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z20, z10, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z14; + z31 = z4; + z14 = _mm512_ternarylogic_epi64(z14, z4, z19, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z19, z9, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z9, z24, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z23; + z31 = z13; + z23 = _mm512_ternarylogic_epi64(z23, z13, z3, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z3, z18, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z18, z8, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z7; + z31 = z22; + z7 = _mm512_ternarylogic_epi64(z7, z22, z12, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z12, z2, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z2, z17, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z16; + z31 = z6; + z16 = _mm512_ternarylogic_epi64(z16, z6, z21, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z21, z11, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z11, z1, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 640))); + /* Round 11 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z14, z23, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z7, z16, 0x96); + z26 = z15; + z26 = _mm512_ternarylogic_epi64(z26, z4, z13, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z22, z6, 0x96); + z27 = z5; + z27 = _mm512_ternarylogic_epi64(z27, z19, z3, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z12, z21, 0x96); + z28 = z20; + z28 = _mm512_ternarylogic_epi64(z28, z9, z18, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z2, z11, 0x96); + z29 = z10; + z29 = _mm512_ternarylogic_epi64(z29, z24, z8, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z17, z1, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z14 = _mm512_xor_si512(z14, z30); + z23 = _mm512_xor_si512(z23, z30); + z7 = _mm512_xor_si512(z7, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z15 = _mm512_xor_si512(z15, z30); + z4 = _mm512_xor_si512(z4, z30); + z13 = _mm512_xor_si512(z13, z30); + z22 = _mm512_xor_si512(z22, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z5 = _mm512_xor_si512(z5, z30); + z19 = _mm512_xor_si512(z19, z30); + z3 = _mm512_xor_si512(z3, z30); + z12 = _mm512_xor_si512(z12, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z20 = _mm512_xor_si512(z20, z30); + z9 = _mm512_xor_si512(z9, z30); + z18 = _mm512_xor_si512(z18, z30); + z2 = _mm512_xor_si512(z2, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z10 = _mm512_xor_si512(z10, z30); + z24 = _mm512_xor_si512(z24, z30); + z8 = _mm512_xor_si512(z8, z30); + z17 = _mm512_xor_si512(z17, z30); + z1 = _mm512_xor_si512(z1, z30); + /* Rho - rotate each lane in place */ + z15 = _mm512_rol_epi64(z15, 1); + z5 = _mm512_rol_epi64(z5, 62); + z20 = _mm512_rol_epi64(z20, 28); + z10 = _mm512_rol_epi64(z10, 27); + z14 = _mm512_rol_epi64(z14, 36); + z4 = _mm512_rol_epi64(z4, 44); + z19 = _mm512_rol_epi64(z19, 6); + z9 = _mm512_rol_epi64(z9, 55); + z24 = _mm512_rol_epi64(z24, 20); + z23 = _mm512_rol_epi64(z23, 3); + z13 = _mm512_rol_epi64(z13, 10); + z3 = _mm512_rol_epi64(z3, 43); + z18 = _mm512_rol_epi64(z18, 25); + z8 = _mm512_rol_epi64(z8, 39); + z7 = _mm512_rol_epi64(z7, 41); + z22 = _mm512_rol_epi64(z22, 45); + z12 = _mm512_rol_epi64(z12, 15); + z2 = _mm512_rol_epi64(z2, 21); + z17 = _mm512_rol_epi64(z17, 8); + z16 = _mm512_rol_epi64(z16, 18); + z6 = _mm512_rol_epi64(z6, 2); + z21 = _mm512_rol_epi64(z21, 61); + z11 = _mm512_rol_epi64(z11, 56); + z1 = _mm512_rol_epi64(z1, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z4; + z0 = _mm512_ternarylogic_epi64(z0, z4, z3, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z3, z2, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z2, z1, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z20; + z31 = z24; + z20 = _mm512_ternarylogic_epi64(z20, z24, z23, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z23, z22, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z22, z21, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z15; + z31 = z19; + z15 = _mm512_ternarylogic_epi64(z15, z19, z18, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z18, z17, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z17, z16, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z10; + z31 = z14; + z10 = _mm512_ternarylogic_epi64(z10, z14, z13, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z13, z12, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z12, z11, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z5; + z31 = z9; + z5 = _mm512_ternarylogic_epi64(z5, z9, z8, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z8, z7, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z7, z6, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 704))); + /* Round 12 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z20, z15, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z10, z5, 0x96); + z26 = z4; + z26 = _mm512_ternarylogic_epi64(z26, z24, z19, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z14, z9, 0x96); + z27 = z3; + z27 = _mm512_ternarylogic_epi64(z27, z23, z18, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z13, z8, 0x96); + z28 = z2; + z28 = _mm512_ternarylogic_epi64(z28, z22, z17, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z12, z7, 0x96); + z29 = z1; + z29 = _mm512_ternarylogic_epi64(z29, z21, z16, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z11, z6, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z20 = _mm512_xor_si512(z20, z30); + z15 = _mm512_xor_si512(z15, z30); + z10 = _mm512_xor_si512(z10, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z4 = _mm512_xor_si512(z4, z30); + z24 = _mm512_xor_si512(z24, z30); + z19 = _mm512_xor_si512(z19, z30); + z14 = _mm512_xor_si512(z14, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z3 = _mm512_xor_si512(z3, z30); + z23 = _mm512_xor_si512(z23, z30); + z18 = _mm512_xor_si512(z18, z30); + z13 = _mm512_xor_si512(z13, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z2 = _mm512_xor_si512(z2, z30); + z22 = _mm512_xor_si512(z22, z30); + z17 = _mm512_xor_si512(z17, z30); + z12 = _mm512_xor_si512(z12, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z1 = _mm512_xor_si512(z1, z30); + z21 = _mm512_xor_si512(z21, z30); + z16 = _mm512_xor_si512(z16, z30); + z11 = _mm512_xor_si512(z11, z30); + z6 = _mm512_xor_si512(z6, z30); + /* Rho - rotate each lane in place */ + z4 = _mm512_rol_epi64(z4, 1); + z3 = _mm512_rol_epi64(z3, 62); + z2 = _mm512_rol_epi64(z2, 28); + z1 = _mm512_rol_epi64(z1, 27); + z20 = _mm512_rol_epi64(z20, 36); + z24 = _mm512_rol_epi64(z24, 44); + z23 = _mm512_rol_epi64(z23, 6); + z22 = _mm512_rol_epi64(z22, 55); + z21 = _mm512_rol_epi64(z21, 20); + z15 = _mm512_rol_epi64(z15, 3); + z19 = _mm512_rol_epi64(z19, 10); + z18 = _mm512_rol_epi64(z18, 43); + z17 = _mm512_rol_epi64(z17, 25); + z16 = _mm512_rol_epi64(z16, 39); + z10 = _mm512_rol_epi64(z10, 41); + z14 = _mm512_rol_epi64(z14, 45); + z13 = _mm512_rol_epi64(z13, 15); + z12 = _mm512_rol_epi64(z12, 21); + z11 = _mm512_rol_epi64(z11, 8); + z5 = _mm512_rol_epi64(z5, 18); + z9 = _mm512_rol_epi64(z9, 2); + z8 = _mm512_rol_epi64(z8, 61); + z7 = _mm512_rol_epi64(z7, 56); + z6 = _mm512_rol_epi64(z6, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z24; + z0 = _mm512_ternarylogic_epi64(z0, z24, z18, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z18, z12, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z12, z6, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z2; + z31 = z21; + z2 = _mm512_ternarylogic_epi64(z2, z21, z15, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z15, z14, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z14, z8, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z4; + z31 = z23; + z4 = _mm512_ternarylogic_epi64(z4, z23, z17, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z17, z11, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z11, z5, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z1; + z31 = z20; + z1 = _mm512_ternarylogic_epi64(z1, z20, z19, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z19, z13, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z13, z7, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z3; + z31 = z22; + z3 = _mm512_ternarylogic_epi64(z3, z22, z16, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z16, z10, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z10, z9, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 768))); + /* Round 13 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z2, z4, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z1, z3, 0x96); + z26 = z24; + z26 = _mm512_ternarylogic_epi64(z26, z21, z23, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z20, z22, 0x96); + z27 = z18; + z27 = _mm512_ternarylogic_epi64(z27, z15, z17, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z19, z16, 0x96); + z28 = z12; + z28 = _mm512_ternarylogic_epi64(z28, z14, z11, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z13, z10, 0x96); + z29 = z6; + z29 = _mm512_ternarylogic_epi64(z29, z8, z5, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z7, z9, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z2 = _mm512_xor_si512(z2, z30); + z4 = _mm512_xor_si512(z4, z30); + z1 = _mm512_xor_si512(z1, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z24 = _mm512_xor_si512(z24, z30); + z21 = _mm512_xor_si512(z21, z30); + z23 = _mm512_xor_si512(z23, z30); + z20 = _mm512_xor_si512(z20, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z18 = _mm512_xor_si512(z18, z30); + z15 = _mm512_xor_si512(z15, z30); + z17 = _mm512_xor_si512(z17, z30); + z19 = _mm512_xor_si512(z19, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z12 = _mm512_xor_si512(z12, z30); + z14 = _mm512_xor_si512(z14, z30); + z11 = _mm512_xor_si512(z11, z30); + z13 = _mm512_xor_si512(z13, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z6 = _mm512_xor_si512(z6, z30); + z8 = _mm512_xor_si512(z8, z30); + z5 = _mm512_xor_si512(z5, z30); + z7 = _mm512_xor_si512(z7, z30); + z9 = _mm512_xor_si512(z9, z30); + /* Rho - rotate each lane in place */ + z24 = _mm512_rol_epi64(z24, 1); + z18 = _mm512_rol_epi64(z18, 62); + z12 = _mm512_rol_epi64(z12, 28); + z6 = _mm512_rol_epi64(z6, 27); + z2 = _mm512_rol_epi64(z2, 36); + z21 = _mm512_rol_epi64(z21, 44); + z15 = _mm512_rol_epi64(z15, 6); + z14 = _mm512_rol_epi64(z14, 55); + z8 = _mm512_rol_epi64(z8, 20); + z4 = _mm512_rol_epi64(z4, 3); + z23 = _mm512_rol_epi64(z23, 10); + z17 = _mm512_rol_epi64(z17, 43); + z11 = _mm512_rol_epi64(z11, 25); + z5 = _mm512_rol_epi64(z5, 39); + z1 = _mm512_rol_epi64(z1, 41); + z20 = _mm512_rol_epi64(z20, 45); + z19 = _mm512_rol_epi64(z19, 15); + z13 = _mm512_rol_epi64(z13, 21); + z7 = _mm512_rol_epi64(z7, 8); + z3 = _mm512_rol_epi64(z3, 18); + z22 = _mm512_rol_epi64(z22, 2); + z16 = _mm512_rol_epi64(z16, 61); + z10 = _mm512_rol_epi64(z10, 56); + z9 = _mm512_rol_epi64(z9, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z21; + z0 = _mm512_ternarylogic_epi64(z0, z21, z17, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z17, z13, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z13, z9, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z12; + z31 = z8; + z12 = _mm512_ternarylogic_epi64(z12, z8, z4, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z4, z20, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z20, z16, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z24; + z31 = z15; + z24 = _mm512_ternarylogic_epi64(z24, z15, z11, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z11, z7, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z7, z3, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z6; + z31 = z2; + z6 = _mm512_ternarylogic_epi64(z6, z2, z23, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z23, z19, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z19, z10, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z18; + z31 = z14; + z18 = _mm512_ternarylogic_epi64(z18, z14, z5, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z5, z1, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z1, z22, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 832))); + /* Round 14 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z12, z24, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z6, z18, 0x96); + z26 = z21; + z26 = _mm512_ternarylogic_epi64(z26, z8, z15, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z2, z14, 0x96); + z27 = z17; + z27 = _mm512_ternarylogic_epi64(z27, z4, z11, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z23, z5, 0x96); + z28 = z13; + z28 = _mm512_ternarylogic_epi64(z28, z20, z7, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z19, z1, 0x96); + z29 = z9; + z29 = _mm512_ternarylogic_epi64(z29, z16, z3, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z10, z22, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z12 = _mm512_xor_si512(z12, z30); + z24 = _mm512_xor_si512(z24, z30); + z6 = _mm512_xor_si512(z6, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z21 = _mm512_xor_si512(z21, z30); + z8 = _mm512_xor_si512(z8, z30); + z15 = _mm512_xor_si512(z15, z30); + z2 = _mm512_xor_si512(z2, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z17 = _mm512_xor_si512(z17, z30); + z4 = _mm512_xor_si512(z4, z30); + z11 = _mm512_xor_si512(z11, z30); + z23 = _mm512_xor_si512(z23, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z13 = _mm512_xor_si512(z13, z30); + z20 = _mm512_xor_si512(z20, z30); + z7 = _mm512_xor_si512(z7, z30); + z19 = _mm512_xor_si512(z19, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z9 = _mm512_xor_si512(z9, z30); + z16 = _mm512_xor_si512(z16, z30); + z3 = _mm512_xor_si512(z3, z30); + z10 = _mm512_xor_si512(z10, z30); + z22 = _mm512_xor_si512(z22, z30); + /* Rho - rotate each lane in place */ + z21 = _mm512_rol_epi64(z21, 1); + z17 = _mm512_rol_epi64(z17, 62); + z13 = _mm512_rol_epi64(z13, 28); + z9 = _mm512_rol_epi64(z9, 27); + z12 = _mm512_rol_epi64(z12, 36); + z8 = _mm512_rol_epi64(z8, 44); + z4 = _mm512_rol_epi64(z4, 6); + z20 = _mm512_rol_epi64(z20, 55); + z16 = _mm512_rol_epi64(z16, 20); + z24 = _mm512_rol_epi64(z24, 3); + z15 = _mm512_rol_epi64(z15, 10); + z11 = _mm512_rol_epi64(z11, 43); + z7 = _mm512_rol_epi64(z7, 25); + z3 = _mm512_rol_epi64(z3, 39); + z6 = _mm512_rol_epi64(z6, 41); + z2 = _mm512_rol_epi64(z2, 45); + z23 = _mm512_rol_epi64(z23, 15); + z19 = _mm512_rol_epi64(z19, 21); + z10 = _mm512_rol_epi64(z10, 8); + z18 = _mm512_rol_epi64(z18, 18); + z14 = _mm512_rol_epi64(z14, 2); + z5 = _mm512_rol_epi64(z5, 61); + z1 = _mm512_rol_epi64(z1, 56); + z22 = _mm512_rol_epi64(z22, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z8; + z0 = _mm512_ternarylogic_epi64(z0, z8, z11, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z11, z19, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z19, z22, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z13; + z31 = z16; + z13 = _mm512_ternarylogic_epi64(z13, z16, z24, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z24, z2, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z2, z5, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z21; + z31 = z4; + z21 = _mm512_ternarylogic_epi64(z21, z4, z7, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z7, z10, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z10, z18, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z9; + z31 = z12; + z9 = _mm512_ternarylogic_epi64(z9, z12, z15, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z15, z23, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z23, z1, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z17; + z31 = z20; + z17 = _mm512_ternarylogic_epi64(z17, z20, z3, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z3, z6, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z6, z14, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 896))); + /* Round 15 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z13, z21, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z9, z17, 0x96); + z26 = z8; + z26 = _mm512_ternarylogic_epi64(z26, z16, z4, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z12, z20, 0x96); + z27 = z11; + z27 = _mm512_ternarylogic_epi64(z27, z24, z7, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z15, z3, 0x96); + z28 = z19; + z28 = _mm512_ternarylogic_epi64(z28, z2, z10, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z23, z6, 0x96); + z29 = z22; + z29 = _mm512_ternarylogic_epi64(z29, z5, z18, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z1, z14, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z13 = _mm512_xor_si512(z13, z30); + z21 = _mm512_xor_si512(z21, z30); + z9 = _mm512_xor_si512(z9, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z8 = _mm512_xor_si512(z8, z30); + z16 = _mm512_xor_si512(z16, z30); + z4 = _mm512_xor_si512(z4, z30); + z12 = _mm512_xor_si512(z12, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z11 = _mm512_xor_si512(z11, z30); + z24 = _mm512_xor_si512(z24, z30); + z7 = _mm512_xor_si512(z7, z30); + z15 = _mm512_xor_si512(z15, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z19 = _mm512_xor_si512(z19, z30); + z2 = _mm512_xor_si512(z2, z30); + z10 = _mm512_xor_si512(z10, z30); + z23 = _mm512_xor_si512(z23, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z22 = _mm512_xor_si512(z22, z30); + z5 = _mm512_xor_si512(z5, z30); + z18 = _mm512_xor_si512(z18, z30); + z1 = _mm512_xor_si512(z1, z30); + z14 = _mm512_xor_si512(z14, z30); + /* Rho - rotate each lane in place */ + z8 = _mm512_rol_epi64(z8, 1); + z11 = _mm512_rol_epi64(z11, 62); + z19 = _mm512_rol_epi64(z19, 28); + z22 = _mm512_rol_epi64(z22, 27); + z13 = _mm512_rol_epi64(z13, 36); + z16 = _mm512_rol_epi64(z16, 44); + z24 = _mm512_rol_epi64(z24, 6); + z2 = _mm512_rol_epi64(z2, 55); + z5 = _mm512_rol_epi64(z5, 20); + z21 = _mm512_rol_epi64(z21, 3); + z4 = _mm512_rol_epi64(z4, 10); + z7 = _mm512_rol_epi64(z7, 43); + z10 = _mm512_rol_epi64(z10, 25); + z18 = _mm512_rol_epi64(z18, 39); + z9 = _mm512_rol_epi64(z9, 41); + z12 = _mm512_rol_epi64(z12, 45); + z15 = _mm512_rol_epi64(z15, 15); + z23 = _mm512_rol_epi64(z23, 21); + z1 = _mm512_rol_epi64(z1, 8); + z17 = _mm512_rol_epi64(z17, 18); + z20 = _mm512_rol_epi64(z20, 2); + z3 = _mm512_rol_epi64(z3, 61); + z6 = _mm512_rol_epi64(z6, 56); + z14 = _mm512_rol_epi64(z14, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z16; + z0 = _mm512_ternarylogic_epi64(z0, z16, z7, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z7, z23, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z23, z14, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z19; + z31 = z5; + z19 = _mm512_ternarylogic_epi64(z19, z5, z21, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z21, z12, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z12, z3, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z8; + z31 = z24; + z8 = _mm512_ternarylogic_epi64(z8, z24, z10, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z10, z1, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z1, z17, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z22; + z31 = z13; + z22 = _mm512_ternarylogic_epi64(z22, z13, z4, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z4, z15, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z15, z6, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z11; + z31 = z2; + z11 = _mm512_ternarylogic_epi64(z11, z2, z18, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z18, z9, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z9, z20, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 960))); + /* Round 16 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z19, z8, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z22, z11, 0x96); + z26 = z16; + z26 = _mm512_ternarylogic_epi64(z26, z5, z24, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z13, z2, 0x96); + z27 = z7; + z27 = _mm512_ternarylogic_epi64(z27, z21, z10, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z4, z18, 0x96); + z28 = z23; + z28 = _mm512_ternarylogic_epi64(z28, z12, z1, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z15, z9, 0x96); + z29 = z14; + z29 = _mm512_ternarylogic_epi64(z29, z3, z17, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z6, z20, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z19 = _mm512_xor_si512(z19, z30); + z8 = _mm512_xor_si512(z8, z30); + z22 = _mm512_xor_si512(z22, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z16 = _mm512_xor_si512(z16, z30); + z5 = _mm512_xor_si512(z5, z30); + z24 = _mm512_xor_si512(z24, z30); + z13 = _mm512_xor_si512(z13, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z7 = _mm512_xor_si512(z7, z30); + z21 = _mm512_xor_si512(z21, z30); + z10 = _mm512_xor_si512(z10, z30); + z4 = _mm512_xor_si512(z4, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z23 = _mm512_xor_si512(z23, z30); + z12 = _mm512_xor_si512(z12, z30); + z1 = _mm512_xor_si512(z1, z30); + z15 = _mm512_xor_si512(z15, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z14 = _mm512_xor_si512(z14, z30); + z3 = _mm512_xor_si512(z3, z30); + z17 = _mm512_xor_si512(z17, z30); + z6 = _mm512_xor_si512(z6, z30); + z20 = _mm512_xor_si512(z20, z30); + /* Rho - rotate each lane in place */ + z16 = _mm512_rol_epi64(z16, 1); + z7 = _mm512_rol_epi64(z7, 62); + z23 = _mm512_rol_epi64(z23, 28); + z14 = _mm512_rol_epi64(z14, 27); + z19 = _mm512_rol_epi64(z19, 36); + z5 = _mm512_rol_epi64(z5, 44); + z21 = _mm512_rol_epi64(z21, 6); + z12 = _mm512_rol_epi64(z12, 55); + z3 = _mm512_rol_epi64(z3, 20); + z8 = _mm512_rol_epi64(z8, 3); + z24 = _mm512_rol_epi64(z24, 10); + z10 = _mm512_rol_epi64(z10, 43); + z1 = _mm512_rol_epi64(z1, 25); + z17 = _mm512_rol_epi64(z17, 39); + z22 = _mm512_rol_epi64(z22, 41); + z13 = _mm512_rol_epi64(z13, 45); + z4 = _mm512_rol_epi64(z4, 15); + z15 = _mm512_rol_epi64(z15, 21); + z6 = _mm512_rol_epi64(z6, 8); + z11 = _mm512_rol_epi64(z11, 18); + z2 = _mm512_rol_epi64(z2, 2); + z18 = _mm512_rol_epi64(z18, 61); + z9 = _mm512_rol_epi64(z9, 56); + z20 = _mm512_rol_epi64(z20, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z5; + z0 = _mm512_ternarylogic_epi64(z0, z5, z10, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z10, z15, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z15, z20, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z23; + z31 = z3; + z23 = _mm512_ternarylogic_epi64(z23, z3, z8, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z8, z13, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z13, z18, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z16; + z31 = z21; + z16 = _mm512_ternarylogic_epi64(z16, z21, z1, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z1, z6, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z6, z11, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z14; + z31 = z19; + z14 = _mm512_ternarylogic_epi64(z14, z19, z24, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z24, z4, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z4, z9, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z7; + z31 = z12; + z7 = _mm512_ternarylogic_epi64(z7, z12, z17, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z17, z22, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z22, z2, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1024))); + /* Round 17 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z23, z16, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z14, z7, 0x96); + z26 = z5; + z26 = _mm512_ternarylogic_epi64(z26, z3, z21, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z19, z12, 0x96); + z27 = z10; + z27 = _mm512_ternarylogic_epi64(z27, z8, z1, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z24, z17, 0x96); + z28 = z15; + z28 = _mm512_ternarylogic_epi64(z28, z13, z6, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z4, z22, 0x96); + z29 = z20; + z29 = _mm512_ternarylogic_epi64(z29, z18, z11, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z9, z2, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z23 = _mm512_xor_si512(z23, z30); + z16 = _mm512_xor_si512(z16, z30); + z14 = _mm512_xor_si512(z14, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z5 = _mm512_xor_si512(z5, z30); + z3 = _mm512_xor_si512(z3, z30); + z21 = _mm512_xor_si512(z21, z30); + z19 = _mm512_xor_si512(z19, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z10 = _mm512_xor_si512(z10, z30); + z8 = _mm512_xor_si512(z8, z30); + z1 = _mm512_xor_si512(z1, z30); + z24 = _mm512_xor_si512(z24, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z15 = _mm512_xor_si512(z15, z30); + z13 = _mm512_xor_si512(z13, z30); + z6 = _mm512_xor_si512(z6, z30); + z4 = _mm512_xor_si512(z4, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z20 = _mm512_xor_si512(z20, z30); + z18 = _mm512_xor_si512(z18, z30); + z11 = _mm512_xor_si512(z11, z30); + z9 = _mm512_xor_si512(z9, z30); + z2 = _mm512_xor_si512(z2, z30); + /* Rho - rotate each lane in place */ + z5 = _mm512_rol_epi64(z5, 1); + z10 = _mm512_rol_epi64(z10, 62); + z15 = _mm512_rol_epi64(z15, 28); + z20 = _mm512_rol_epi64(z20, 27); + z23 = _mm512_rol_epi64(z23, 36); + z3 = _mm512_rol_epi64(z3, 44); + z8 = _mm512_rol_epi64(z8, 6); + z13 = _mm512_rol_epi64(z13, 55); + z18 = _mm512_rol_epi64(z18, 20); + z16 = _mm512_rol_epi64(z16, 3); + z21 = _mm512_rol_epi64(z21, 10); + z1 = _mm512_rol_epi64(z1, 43); + z6 = _mm512_rol_epi64(z6, 25); + z11 = _mm512_rol_epi64(z11, 39); + z14 = _mm512_rol_epi64(z14, 41); + z19 = _mm512_rol_epi64(z19, 45); + z24 = _mm512_rol_epi64(z24, 15); + z4 = _mm512_rol_epi64(z4, 21); + z9 = _mm512_rol_epi64(z9, 8); + z7 = _mm512_rol_epi64(z7, 18); + z12 = _mm512_rol_epi64(z12, 2); + z17 = _mm512_rol_epi64(z17, 61); + z22 = _mm512_rol_epi64(z22, 56); + z2 = _mm512_rol_epi64(z2, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z3; + z0 = _mm512_ternarylogic_epi64(z0, z3, z1, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z1, z4, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z4, z2, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z15; + z31 = z18; + z15 = _mm512_ternarylogic_epi64(z15, z18, z16, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z16, z19, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z19, z17, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z5; + z31 = z8; + z5 = _mm512_ternarylogic_epi64(z5, z8, z6, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z6, z9, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z9, z7, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z20; + z31 = z23; + z20 = _mm512_ternarylogic_epi64(z20, z23, z21, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z21, z24, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z24, z22, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z10; + z31 = z13; + z10 = _mm512_ternarylogic_epi64(z10, z13, z11, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z11, z14, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z14, z12, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1088))); + /* Round 18 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z15, z5, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z20, z10, 0x96); + z26 = z3; + z26 = _mm512_ternarylogic_epi64(z26, z18, z8, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z23, z13, 0x96); + z27 = z1; + z27 = _mm512_ternarylogic_epi64(z27, z16, z6, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z21, z11, 0x96); + z28 = z4; + z28 = _mm512_ternarylogic_epi64(z28, z19, z9, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z24, z14, 0x96); + z29 = z2; + z29 = _mm512_ternarylogic_epi64(z29, z17, z7, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z22, z12, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z15 = _mm512_xor_si512(z15, z30); + z5 = _mm512_xor_si512(z5, z30); + z20 = _mm512_xor_si512(z20, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z3 = _mm512_xor_si512(z3, z30); + z18 = _mm512_xor_si512(z18, z30); + z8 = _mm512_xor_si512(z8, z30); + z23 = _mm512_xor_si512(z23, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z1 = _mm512_xor_si512(z1, z30); + z16 = _mm512_xor_si512(z16, z30); + z6 = _mm512_xor_si512(z6, z30); + z21 = _mm512_xor_si512(z21, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z4 = _mm512_xor_si512(z4, z30); + z19 = _mm512_xor_si512(z19, z30); + z9 = _mm512_xor_si512(z9, z30); + z24 = _mm512_xor_si512(z24, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z2 = _mm512_xor_si512(z2, z30); + z17 = _mm512_xor_si512(z17, z30); + z7 = _mm512_xor_si512(z7, z30); + z22 = _mm512_xor_si512(z22, z30); + z12 = _mm512_xor_si512(z12, z30); + /* Rho - rotate each lane in place */ + z3 = _mm512_rol_epi64(z3, 1); + z1 = _mm512_rol_epi64(z1, 62); + z4 = _mm512_rol_epi64(z4, 28); + z2 = _mm512_rol_epi64(z2, 27); + z15 = _mm512_rol_epi64(z15, 36); + z18 = _mm512_rol_epi64(z18, 44); + z16 = _mm512_rol_epi64(z16, 6); + z19 = _mm512_rol_epi64(z19, 55); + z17 = _mm512_rol_epi64(z17, 20); + z5 = _mm512_rol_epi64(z5, 3); + z8 = _mm512_rol_epi64(z8, 10); + z6 = _mm512_rol_epi64(z6, 43); + z9 = _mm512_rol_epi64(z9, 25); + z7 = _mm512_rol_epi64(z7, 39); + z20 = _mm512_rol_epi64(z20, 41); + z23 = _mm512_rol_epi64(z23, 45); + z21 = _mm512_rol_epi64(z21, 15); + z24 = _mm512_rol_epi64(z24, 21); + z22 = _mm512_rol_epi64(z22, 8); + z10 = _mm512_rol_epi64(z10, 18); + z13 = _mm512_rol_epi64(z13, 2); + z11 = _mm512_rol_epi64(z11, 61); + z14 = _mm512_rol_epi64(z14, 56); + z12 = _mm512_rol_epi64(z12, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z18; + z0 = _mm512_ternarylogic_epi64(z0, z18, z6, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z6, z24, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z24, z12, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z4; + z31 = z17; + z4 = _mm512_ternarylogic_epi64(z4, z17, z5, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z5, z23, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z23, z11, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z3; + z31 = z16; + z3 = _mm512_ternarylogic_epi64(z3, z16, z9, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z9, z22, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z22, z10, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z2; + z31 = z15; + z2 = _mm512_ternarylogic_epi64(z2, z15, z8, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z8, z21, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z21, z14, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z1; + z31 = z19; + z1 = _mm512_ternarylogic_epi64(z1, z19, z7, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z7, z20, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z20, z13, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1152))); + /* Round 19 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z4, z3, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z2, z1, 0x96); + z26 = z18; + z26 = _mm512_ternarylogic_epi64(z26, z17, z16, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z15, z19, 0x96); + z27 = z6; + z27 = _mm512_ternarylogic_epi64(z27, z5, z9, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z8, z7, 0x96); + z28 = z24; + z28 = _mm512_ternarylogic_epi64(z28, z23, z22, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z21, z20, 0x96); + z29 = z12; + z29 = _mm512_ternarylogic_epi64(z29, z11, z10, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z14, z13, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z4 = _mm512_xor_si512(z4, z30); + z3 = _mm512_xor_si512(z3, z30); + z2 = _mm512_xor_si512(z2, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z18 = _mm512_xor_si512(z18, z30); + z17 = _mm512_xor_si512(z17, z30); + z16 = _mm512_xor_si512(z16, z30); + z15 = _mm512_xor_si512(z15, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z6 = _mm512_xor_si512(z6, z30); + z5 = _mm512_xor_si512(z5, z30); + z9 = _mm512_xor_si512(z9, z30); + z8 = _mm512_xor_si512(z8, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z24 = _mm512_xor_si512(z24, z30); + z23 = _mm512_xor_si512(z23, z30); + z22 = _mm512_xor_si512(z22, z30); + z21 = _mm512_xor_si512(z21, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z12 = _mm512_xor_si512(z12, z30); + z11 = _mm512_xor_si512(z11, z30); + z10 = _mm512_xor_si512(z10, z30); + z14 = _mm512_xor_si512(z14, z30); + z13 = _mm512_xor_si512(z13, z30); + /* Rho - rotate each lane in place */ + z18 = _mm512_rol_epi64(z18, 1); + z6 = _mm512_rol_epi64(z6, 62); + z24 = _mm512_rol_epi64(z24, 28); + z12 = _mm512_rol_epi64(z12, 27); + z4 = _mm512_rol_epi64(z4, 36); + z17 = _mm512_rol_epi64(z17, 44); + z5 = _mm512_rol_epi64(z5, 6); + z23 = _mm512_rol_epi64(z23, 55); + z11 = _mm512_rol_epi64(z11, 20); + z3 = _mm512_rol_epi64(z3, 3); + z16 = _mm512_rol_epi64(z16, 10); + z9 = _mm512_rol_epi64(z9, 43); + z22 = _mm512_rol_epi64(z22, 25); + z10 = _mm512_rol_epi64(z10, 39); + z2 = _mm512_rol_epi64(z2, 41); + z15 = _mm512_rol_epi64(z15, 45); + z8 = _mm512_rol_epi64(z8, 15); + z21 = _mm512_rol_epi64(z21, 21); + z14 = _mm512_rol_epi64(z14, 8); + z1 = _mm512_rol_epi64(z1, 18); + z19 = _mm512_rol_epi64(z19, 2); + z7 = _mm512_rol_epi64(z7, 61); + z20 = _mm512_rol_epi64(z20, 56); + z13 = _mm512_rol_epi64(z13, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z17; + z0 = _mm512_ternarylogic_epi64(z0, z17, z9, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z9, z21, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z21, z13, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z24; + z31 = z11; + z24 = _mm512_ternarylogic_epi64(z24, z11, z3, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z3, z15, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z15, z7, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z18; + z31 = z5; + z18 = _mm512_ternarylogic_epi64(z18, z5, z22, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z22, z14, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z14, z1, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z12; + z31 = z4; + z12 = _mm512_ternarylogic_epi64(z12, z4, z16, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z16, z8, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z8, z20, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z6; + z31 = z23; + z6 = _mm512_ternarylogic_epi64(z6, z23, z10, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z10, z2, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z2, z19, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1216))); + /* Round 20 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z24, z18, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z12, z6, 0x96); + z26 = z17; + z26 = _mm512_ternarylogic_epi64(z26, z11, z5, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z4, z23, 0x96); + z27 = z9; + z27 = _mm512_ternarylogic_epi64(z27, z3, z22, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z16, z10, 0x96); + z28 = z21; + z28 = _mm512_ternarylogic_epi64(z28, z15, z14, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z8, z2, 0x96); + z29 = z13; + z29 = _mm512_ternarylogic_epi64(z29, z7, z1, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z20, z19, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z24 = _mm512_xor_si512(z24, z30); + z18 = _mm512_xor_si512(z18, z30); + z12 = _mm512_xor_si512(z12, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z17 = _mm512_xor_si512(z17, z30); + z11 = _mm512_xor_si512(z11, z30); + z5 = _mm512_xor_si512(z5, z30); + z4 = _mm512_xor_si512(z4, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z9 = _mm512_xor_si512(z9, z30); + z3 = _mm512_xor_si512(z3, z30); + z22 = _mm512_xor_si512(z22, z30); + z16 = _mm512_xor_si512(z16, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z21 = _mm512_xor_si512(z21, z30); + z15 = _mm512_xor_si512(z15, z30); + z14 = _mm512_xor_si512(z14, z30); + z8 = _mm512_xor_si512(z8, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z13 = _mm512_xor_si512(z13, z30); + z7 = _mm512_xor_si512(z7, z30); + z1 = _mm512_xor_si512(z1, z30); + z20 = _mm512_xor_si512(z20, z30); + z19 = _mm512_xor_si512(z19, z30); + /* Rho - rotate each lane in place */ + z17 = _mm512_rol_epi64(z17, 1); + z9 = _mm512_rol_epi64(z9, 62); + z21 = _mm512_rol_epi64(z21, 28); + z13 = _mm512_rol_epi64(z13, 27); + z24 = _mm512_rol_epi64(z24, 36); + z11 = _mm512_rol_epi64(z11, 44); + z3 = _mm512_rol_epi64(z3, 6); + z15 = _mm512_rol_epi64(z15, 55); + z7 = _mm512_rol_epi64(z7, 20); + z18 = _mm512_rol_epi64(z18, 3); + z5 = _mm512_rol_epi64(z5, 10); + z22 = _mm512_rol_epi64(z22, 43); + z14 = _mm512_rol_epi64(z14, 25); + z1 = _mm512_rol_epi64(z1, 39); + z12 = _mm512_rol_epi64(z12, 41); + z4 = _mm512_rol_epi64(z4, 45); + z16 = _mm512_rol_epi64(z16, 15); + z8 = _mm512_rol_epi64(z8, 21); + z20 = _mm512_rol_epi64(z20, 8); + z6 = _mm512_rol_epi64(z6, 18); + z23 = _mm512_rol_epi64(z23, 2); + z10 = _mm512_rol_epi64(z10, 61); + z2 = _mm512_rol_epi64(z2, 56); + z19 = _mm512_rol_epi64(z19, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z11; + z0 = _mm512_ternarylogic_epi64(z0, z11, z22, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z22, z8, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z8, z19, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z21; + z31 = z7; + z21 = _mm512_ternarylogic_epi64(z21, z7, z18, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z18, z4, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z4, z10, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z17; + z31 = z3; + z17 = _mm512_ternarylogic_epi64(z17, z3, z14, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z14, z20, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z20, z6, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z13; + z31 = z24; + z13 = _mm512_ternarylogic_epi64(z13, z24, z5, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z5, z16, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z16, z2, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z9; + z31 = z15; + z9 = _mm512_ternarylogic_epi64(z9, z15, z1, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z1, z12, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z12, z23, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1280))); + /* Round 21 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z21, z17, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z13, z9, 0x96); + z26 = z11; + z26 = _mm512_ternarylogic_epi64(z26, z7, z3, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z24, z15, 0x96); + z27 = z22; + z27 = _mm512_ternarylogic_epi64(z27, z18, z14, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z5, z1, 0x96); + z28 = z8; + z28 = _mm512_ternarylogic_epi64(z28, z4, z20, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z16, z12, 0x96); + z29 = z19; + z29 = _mm512_ternarylogic_epi64(z29, z10, z6, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z2, z23, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z21 = _mm512_xor_si512(z21, z30); + z17 = _mm512_xor_si512(z17, z30); + z13 = _mm512_xor_si512(z13, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z11 = _mm512_xor_si512(z11, z30); + z7 = _mm512_xor_si512(z7, z30); + z3 = _mm512_xor_si512(z3, z30); + z24 = _mm512_xor_si512(z24, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z22 = _mm512_xor_si512(z22, z30); + z18 = _mm512_xor_si512(z18, z30); + z14 = _mm512_xor_si512(z14, z30); + z5 = _mm512_xor_si512(z5, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z8 = _mm512_xor_si512(z8, z30); + z4 = _mm512_xor_si512(z4, z30); + z20 = _mm512_xor_si512(z20, z30); + z16 = _mm512_xor_si512(z16, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z19 = _mm512_xor_si512(z19, z30); + z10 = _mm512_xor_si512(z10, z30); + z6 = _mm512_xor_si512(z6, z30); + z2 = _mm512_xor_si512(z2, z30); + z23 = _mm512_xor_si512(z23, z30); + /* Rho - rotate each lane in place */ + z11 = _mm512_rol_epi64(z11, 1); + z22 = _mm512_rol_epi64(z22, 62); + z8 = _mm512_rol_epi64(z8, 28); + z19 = _mm512_rol_epi64(z19, 27); + z21 = _mm512_rol_epi64(z21, 36); + z7 = _mm512_rol_epi64(z7, 44); + z18 = _mm512_rol_epi64(z18, 6); + z4 = _mm512_rol_epi64(z4, 55); + z10 = _mm512_rol_epi64(z10, 20); + z17 = _mm512_rol_epi64(z17, 3); + z3 = _mm512_rol_epi64(z3, 10); + z14 = _mm512_rol_epi64(z14, 43); + z20 = _mm512_rol_epi64(z20, 25); + z6 = _mm512_rol_epi64(z6, 39); + z13 = _mm512_rol_epi64(z13, 41); + z24 = _mm512_rol_epi64(z24, 45); + z5 = _mm512_rol_epi64(z5, 15); + z16 = _mm512_rol_epi64(z16, 21); + z2 = _mm512_rol_epi64(z2, 8); + z9 = _mm512_rol_epi64(z9, 18); + z15 = _mm512_rol_epi64(z15, 2); + z1 = _mm512_rol_epi64(z1, 61); + z12 = _mm512_rol_epi64(z12, 56); + z23 = _mm512_rol_epi64(z23, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z7; + z0 = _mm512_ternarylogic_epi64(z0, z7, z14, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z14, z16, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z16, z23, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z8; + z31 = z10; + z8 = _mm512_ternarylogic_epi64(z8, z10, z17, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z17, z24, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z24, z1, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z11; + z31 = z18; + z11 = _mm512_ternarylogic_epi64(z11, z18, z20, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z20, z2, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z2, z9, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z19; + z31 = z21; + z19 = _mm512_ternarylogic_epi64(z19, z21, z3, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z3, z5, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z5, z12, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z22; + z31 = z4; + z22 = _mm512_ternarylogic_epi64(z22, z4, z6, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z6, z13, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z13, z15, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1344))); + /* Round 22 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z8, z11, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z19, z22, 0x96); + z26 = z7; + z26 = _mm512_ternarylogic_epi64(z26, z10, z18, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z21, z4, 0x96); + z27 = z14; + z27 = _mm512_ternarylogic_epi64(z27, z17, z20, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z3, z6, 0x96); + z28 = z16; + z28 = _mm512_ternarylogic_epi64(z28, z24, z2, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z5, z13, 0x96); + z29 = z23; + z29 = _mm512_ternarylogic_epi64(z29, z1, z9, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z12, z15, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z8 = _mm512_xor_si512(z8, z30); + z11 = _mm512_xor_si512(z11, z30); + z19 = _mm512_xor_si512(z19, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z7 = _mm512_xor_si512(z7, z30); + z10 = _mm512_xor_si512(z10, z30); + z18 = _mm512_xor_si512(z18, z30); + z21 = _mm512_xor_si512(z21, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z14 = _mm512_xor_si512(z14, z30); + z17 = _mm512_xor_si512(z17, z30); + z20 = _mm512_xor_si512(z20, z30); + z3 = _mm512_xor_si512(z3, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z16 = _mm512_xor_si512(z16, z30); + z24 = _mm512_xor_si512(z24, z30); + z2 = _mm512_xor_si512(z2, z30); + z5 = _mm512_xor_si512(z5, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z23 = _mm512_xor_si512(z23, z30); + z1 = _mm512_xor_si512(z1, z30); + z9 = _mm512_xor_si512(z9, z30); + z12 = _mm512_xor_si512(z12, z30); + z15 = _mm512_xor_si512(z15, z30); + /* Rho - rotate each lane in place */ + z7 = _mm512_rol_epi64(z7, 1); + z14 = _mm512_rol_epi64(z14, 62); + z16 = _mm512_rol_epi64(z16, 28); + z23 = _mm512_rol_epi64(z23, 27); + z8 = _mm512_rol_epi64(z8, 36); + z10 = _mm512_rol_epi64(z10, 44); + z17 = _mm512_rol_epi64(z17, 6); + z24 = _mm512_rol_epi64(z24, 55); + z1 = _mm512_rol_epi64(z1, 20); + z11 = _mm512_rol_epi64(z11, 3); + z18 = _mm512_rol_epi64(z18, 10); + z20 = _mm512_rol_epi64(z20, 43); + z2 = _mm512_rol_epi64(z2, 25); + z9 = _mm512_rol_epi64(z9, 39); + z19 = _mm512_rol_epi64(z19, 41); + z21 = _mm512_rol_epi64(z21, 45); + z3 = _mm512_rol_epi64(z3, 15); + z5 = _mm512_rol_epi64(z5, 21); + z12 = _mm512_rol_epi64(z12, 8); + z22 = _mm512_rol_epi64(z22, 18); + z4 = _mm512_rol_epi64(z4, 2); + z6 = _mm512_rol_epi64(z6, 61); + z13 = _mm512_rol_epi64(z13, 56); + z15 = _mm512_rol_epi64(z15, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z10; + z0 = _mm512_ternarylogic_epi64(z0, z10, z20, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z20, z5, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z5, z15, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z16; + z31 = z1; + z16 = _mm512_ternarylogic_epi64(z16, z1, z11, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z11, z21, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z21, z6, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z7; + z31 = z17; + z7 = _mm512_ternarylogic_epi64(z7, z17, z2, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z2, z12, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z12, z22, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z23; + z31 = z8; + z23 = _mm512_ternarylogic_epi64(z23, z8, z18, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z18, z3, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z3, z13, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z14; + z31 = z24; + z14 = _mm512_ternarylogic_epi64(z14, z24, z9, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z9, z19, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z19, z4, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1408))); + /* Round 23 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z16, z7, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z23, z14, 0x96); + z26 = z10; + z26 = _mm512_ternarylogic_epi64(z26, z1, z17, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z8, z24, 0x96); + z27 = z20; + z27 = _mm512_ternarylogic_epi64(z27, z11, z2, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z18, z9, 0x96); + z28 = z5; + z28 = _mm512_ternarylogic_epi64(z28, z21, z12, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z3, z19, 0x96); + z29 = z15; + z29 = _mm512_ternarylogic_epi64(z29, z6, z22, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z13, z4, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z16 = _mm512_xor_si512(z16, z30); + z7 = _mm512_xor_si512(z7, z30); + z23 = _mm512_xor_si512(z23, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z10 = _mm512_xor_si512(z10, z30); + z1 = _mm512_xor_si512(z1, z30); + z17 = _mm512_xor_si512(z17, z30); + z8 = _mm512_xor_si512(z8, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z20 = _mm512_xor_si512(z20, z30); + z11 = _mm512_xor_si512(z11, z30); + z2 = _mm512_xor_si512(z2, z30); + z18 = _mm512_xor_si512(z18, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z5 = _mm512_xor_si512(z5, z30); + z21 = _mm512_xor_si512(z21, z30); + z12 = _mm512_xor_si512(z12, z30); + z3 = _mm512_xor_si512(z3, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z15 = _mm512_xor_si512(z15, z30); + z6 = _mm512_xor_si512(z6, z30); + z22 = _mm512_xor_si512(z22, z30); + z13 = _mm512_xor_si512(z13, z30); + z4 = _mm512_xor_si512(z4, z30); + /* Rho - rotate each lane in place */ + z10 = _mm512_rol_epi64(z10, 1); + z20 = _mm512_rol_epi64(z20, 62); + z5 = _mm512_rol_epi64(z5, 28); + z15 = _mm512_rol_epi64(z15, 27); + z16 = _mm512_rol_epi64(z16, 36); + z1 = _mm512_rol_epi64(z1, 44); + z11 = _mm512_rol_epi64(z11, 6); + z21 = _mm512_rol_epi64(z21, 55); + z6 = _mm512_rol_epi64(z6, 20); + z7 = _mm512_rol_epi64(z7, 3); + z17 = _mm512_rol_epi64(z17, 10); + z2 = _mm512_rol_epi64(z2, 43); + z12 = _mm512_rol_epi64(z12, 25); + z22 = _mm512_rol_epi64(z22, 39); + z23 = _mm512_rol_epi64(z23, 41); + z8 = _mm512_rol_epi64(z8, 45); + z18 = _mm512_rol_epi64(z18, 15); + z3 = _mm512_rol_epi64(z3, 21); + z13 = _mm512_rol_epi64(z13, 8); + z14 = _mm512_rol_epi64(z14, 18); + z24 = _mm512_rol_epi64(z24, 2); + z9 = _mm512_rol_epi64(z9, 61); + z19 = _mm512_rol_epi64(z19, 56); + z4 = _mm512_rol_epi64(z4, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z1; + z0 = _mm512_ternarylogic_epi64(z0, z1, z2, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z2, z3, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z3, z4, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z5; + z31 = z6; + z5 = _mm512_ternarylogic_epi64(z5, z6, z7, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z7, z8, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z8, z9, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z10; + z31 = z11; + z10 = _mm512_ternarylogic_epi64(z10, z11, z12, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z12, z13, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z13, z14, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z15; + z31 = z16; + z15 = _mm512_ternarylogic_epi64(z15, z16, z17, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z17, z18, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z18, z19, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z20; + z31 = z21; + z20 = _mm512_ternarylogic_epi64(z20, z21, z22, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z22, z23, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z23, z24, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1472))); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 1024), z16); + _mm512_storeu_si512((void*)WC_PW(rdi, 1088), z17); + _mm512_storeu_si512((void*)WC_PW(rdi, 1152), z18); + _mm512_storeu_si512((void*)WC_PW(rdi, 1216), z19); + _mm512_storeu_si512((void*)WC_PW(rdi, 1280), z20); + _mm512_storeu_si512((void*)WC_PW(rdi, 1344), z21); + _mm512_storeu_si512((void*)WC_PW(rdi, 1408), z22); + _mm512_storeu_si512((void*)WC_PW(rdi, 1472), z23); + _mm512_storeu_si512((void*)WC_PW(rdi, 1536), z24); +} + +XALIGNED(32) static const word64 L_sha3_128_blocksx8_seed_avx512_end_mark[] + WC_X64I_UNUSED = { + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void sha3_128_blocksx8_seed_avx512(word64* s, byte* seed) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23, z24, z25, z26, z27, + z28, z29, z30, z31; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)seed; + + rdx = (word64)(L_sha3_x8_avx512_r); + z0 = _mm512_set1_epi64((long long)WC_L64(rsi, 0)); + z1 = _mm512_set1_epi64((long long)WC_L64(rsi, 8)); + z2 = _mm512_set1_epi64((long long)WC_L64(rsi, 16)); + z3 = _mm512_set1_epi64((long long)WC_L64(rsi, 24)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_setzero_si512(); + z6 = z5; + z7 = z5; + z8 = z5; + z9 = z5; + z10 = z5; + z11 = z5; + z12 = z5; + z13 = z5; + z14 = z5; + z15 = z5; + z16 = z5; + z17 = z5; + z18 = z5; + z19 = z5; + z20 = _mm512_loadu_si512((const void*)WC_PR( + L_sha3_128_blocksx8_seed_avx512_end_mark, 0)); + z21 = z5; + z22 = z5; + z23 = z5; + z24 = z5; + /* Round 0 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z5, z10, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z15, z20, 0x96); + z26 = z1; + z26 = _mm512_ternarylogic_epi64(z26, z6, z11, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z16, z21, 0x96); + z27 = z2; + z27 = _mm512_ternarylogic_epi64(z27, z7, z12, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z17, z22, 0x96); + z28 = z3; + z28 = _mm512_ternarylogic_epi64(z28, z8, z13, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z18, z23, 0x96); + z29 = z4; + z29 = _mm512_ternarylogic_epi64(z29, z9, z14, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z19, z24, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z5 = _mm512_xor_si512(z5, z30); + z10 = _mm512_xor_si512(z10, z30); + z15 = _mm512_xor_si512(z15, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z1 = _mm512_xor_si512(z1, z30); + z6 = _mm512_xor_si512(z6, z30); + z11 = _mm512_xor_si512(z11, z30); + z16 = _mm512_xor_si512(z16, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z2 = _mm512_xor_si512(z2, z30); + z7 = _mm512_xor_si512(z7, z30); + z12 = _mm512_xor_si512(z12, z30); + z17 = _mm512_xor_si512(z17, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z3 = _mm512_xor_si512(z3, z30); + z8 = _mm512_xor_si512(z8, z30); + z13 = _mm512_xor_si512(z13, z30); + z18 = _mm512_xor_si512(z18, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z4 = _mm512_xor_si512(z4, z30); + z9 = _mm512_xor_si512(z9, z30); + z14 = _mm512_xor_si512(z14, z30); + z19 = _mm512_xor_si512(z19, z30); + z24 = _mm512_xor_si512(z24, z30); + /* Rho - rotate each lane in place */ + z1 = _mm512_rol_epi64(z1, 1); + z2 = _mm512_rol_epi64(z2, 62); + z3 = _mm512_rol_epi64(z3, 28); + z4 = _mm512_rol_epi64(z4, 27); + z5 = _mm512_rol_epi64(z5, 36); + z6 = _mm512_rol_epi64(z6, 44); + z7 = _mm512_rol_epi64(z7, 6); + z8 = _mm512_rol_epi64(z8, 55); + z9 = _mm512_rol_epi64(z9, 20); + z10 = _mm512_rol_epi64(z10, 3); + z11 = _mm512_rol_epi64(z11, 10); + z12 = _mm512_rol_epi64(z12, 43); + z13 = _mm512_rol_epi64(z13, 25); + z14 = _mm512_rol_epi64(z14, 39); + z15 = _mm512_rol_epi64(z15, 41); + z16 = _mm512_rol_epi64(z16, 45); + z17 = _mm512_rol_epi64(z17, 15); + z18 = _mm512_rol_epi64(z18, 21); + z19 = _mm512_rol_epi64(z19, 8); + z20 = _mm512_rol_epi64(z20, 18); + z21 = _mm512_rol_epi64(z21, 2); + z22 = _mm512_rol_epi64(z22, 61); + z23 = _mm512_rol_epi64(z23, 56); + z24 = _mm512_rol_epi64(z24, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z6; + z0 = _mm512_ternarylogic_epi64(z0, z6, z12, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z12, z18, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z18, z24, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z3; + z31 = z9; + z3 = _mm512_ternarylogic_epi64(z3, z9, z10, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z10, z16, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z16, z22, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z1; + z31 = z7; + z1 = _mm512_ternarylogic_epi64(z1, z7, z13, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z13, z19, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z19, z20, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z4; + z31 = z5; + z4 = _mm512_ternarylogic_epi64(z4, z5, z11, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z11, z17, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z17, z23, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z2; + z31 = z8; + z2 = _mm512_ternarylogic_epi64(z2, z8, z14, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z14, z15, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z15, z21, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 0))); + /* Round 1 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z3, z1, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z4, z2, 0x96); + z26 = z6; + z26 = _mm512_ternarylogic_epi64(z26, z9, z7, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z5, z8, 0x96); + z27 = z12; + z27 = _mm512_ternarylogic_epi64(z27, z10, z13, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z11, z14, 0x96); + z28 = z18; + z28 = _mm512_ternarylogic_epi64(z28, z16, z19, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z17, z15, 0x96); + z29 = z24; + z29 = _mm512_ternarylogic_epi64(z29, z22, z20, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z23, z21, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z3 = _mm512_xor_si512(z3, z30); + z1 = _mm512_xor_si512(z1, z30); + z4 = _mm512_xor_si512(z4, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z6 = _mm512_xor_si512(z6, z30); + z9 = _mm512_xor_si512(z9, z30); + z7 = _mm512_xor_si512(z7, z30); + z5 = _mm512_xor_si512(z5, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z12 = _mm512_xor_si512(z12, z30); + z10 = _mm512_xor_si512(z10, z30); + z13 = _mm512_xor_si512(z13, z30); + z11 = _mm512_xor_si512(z11, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z18 = _mm512_xor_si512(z18, z30); + z16 = _mm512_xor_si512(z16, z30); + z19 = _mm512_xor_si512(z19, z30); + z17 = _mm512_xor_si512(z17, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z24 = _mm512_xor_si512(z24, z30); + z22 = _mm512_xor_si512(z22, z30); + z20 = _mm512_xor_si512(z20, z30); + z23 = _mm512_xor_si512(z23, z30); + z21 = _mm512_xor_si512(z21, z30); + /* Rho - rotate each lane in place */ + z6 = _mm512_rol_epi64(z6, 1); + z12 = _mm512_rol_epi64(z12, 62); + z18 = _mm512_rol_epi64(z18, 28); + z24 = _mm512_rol_epi64(z24, 27); + z3 = _mm512_rol_epi64(z3, 36); + z9 = _mm512_rol_epi64(z9, 44); + z10 = _mm512_rol_epi64(z10, 6); + z16 = _mm512_rol_epi64(z16, 55); + z22 = _mm512_rol_epi64(z22, 20); + z1 = _mm512_rol_epi64(z1, 3); + z7 = _mm512_rol_epi64(z7, 10); + z13 = _mm512_rol_epi64(z13, 43); + z19 = _mm512_rol_epi64(z19, 25); + z20 = _mm512_rol_epi64(z20, 39); + z4 = _mm512_rol_epi64(z4, 41); + z5 = _mm512_rol_epi64(z5, 45); + z11 = _mm512_rol_epi64(z11, 15); + z17 = _mm512_rol_epi64(z17, 21); + z23 = _mm512_rol_epi64(z23, 8); + z2 = _mm512_rol_epi64(z2, 18); + z8 = _mm512_rol_epi64(z8, 2); + z14 = _mm512_rol_epi64(z14, 61); + z15 = _mm512_rol_epi64(z15, 56); + z21 = _mm512_rol_epi64(z21, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z9; + z0 = _mm512_ternarylogic_epi64(z0, z9, z13, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z13, z17, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z17, z21, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z18; + z31 = z22; + z18 = _mm512_ternarylogic_epi64(z18, z22, z1, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z1, z5, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z5, z14, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z6; + z31 = z10; + z6 = _mm512_ternarylogic_epi64(z6, z10, z19, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z19, z23, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z23, z2, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z24; + z31 = z3; + z24 = _mm512_ternarylogic_epi64(z24, z3, z7, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z7, z11, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z11, z15, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z12; + z31 = z16; + z12 = _mm512_ternarylogic_epi64(z12, z16, z20, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z20, z4, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z4, z8, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 64))); + /* Round 2 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z18, z6, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z24, z12, 0x96); + z26 = z9; + z26 = _mm512_ternarylogic_epi64(z26, z22, z10, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z3, z16, 0x96); + z27 = z13; + z27 = _mm512_ternarylogic_epi64(z27, z1, z19, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z7, z20, 0x96); + z28 = z17; + z28 = _mm512_ternarylogic_epi64(z28, z5, z23, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z11, z4, 0x96); + z29 = z21; + z29 = _mm512_ternarylogic_epi64(z29, z14, z2, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z15, z8, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z18 = _mm512_xor_si512(z18, z30); + z6 = _mm512_xor_si512(z6, z30); + z24 = _mm512_xor_si512(z24, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z9 = _mm512_xor_si512(z9, z30); + z22 = _mm512_xor_si512(z22, z30); + z10 = _mm512_xor_si512(z10, z30); + z3 = _mm512_xor_si512(z3, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z13 = _mm512_xor_si512(z13, z30); + z1 = _mm512_xor_si512(z1, z30); + z19 = _mm512_xor_si512(z19, z30); + z7 = _mm512_xor_si512(z7, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z17 = _mm512_xor_si512(z17, z30); + z5 = _mm512_xor_si512(z5, z30); + z23 = _mm512_xor_si512(z23, z30); + z11 = _mm512_xor_si512(z11, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z21 = _mm512_xor_si512(z21, z30); + z14 = _mm512_xor_si512(z14, z30); + z2 = _mm512_xor_si512(z2, z30); + z15 = _mm512_xor_si512(z15, z30); + z8 = _mm512_xor_si512(z8, z30); + /* Rho - rotate each lane in place */ + z9 = _mm512_rol_epi64(z9, 1); + z13 = _mm512_rol_epi64(z13, 62); + z17 = _mm512_rol_epi64(z17, 28); + z21 = _mm512_rol_epi64(z21, 27); + z18 = _mm512_rol_epi64(z18, 36); + z22 = _mm512_rol_epi64(z22, 44); + z1 = _mm512_rol_epi64(z1, 6); + z5 = _mm512_rol_epi64(z5, 55); + z14 = _mm512_rol_epi64(z14, 20); + z6 = _mm512_rol_epi64(z6, 3); + z10 = _mm512_rol_epi64(z10, 10); + z19 = _mm512_rol_epi64(z19, 43); + z23 = _mm512_rol_epi64(z23, 25); + z2 = _mm512_rol_epi64(z2, 39); + z24 = _mm512_rol_epi64(z24, 41); + z3 = _mm512_rol_epi64(z3, 45); + z7 = _mm512_rol_epi64(z7, 15); + z11 = _mm512_rol_epi64(z11, 21); + z15 = _mm512_rol_epi64(z15, 8); + z12 = _mm512_rol_epi64(z12, 18); + z16 = _mm512_rol_epi64(z16, 2); + z20 = _mm512_rol_epi64(z20, 61); + z4 = _mm512_rol_epi64(z4, 56); + z8 = _mm512_rol_epi64(z8, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z22; + z0 = _mm512_ternarylogic_epi64(z0, z22, z19, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z19, z11, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z11, z8, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z17; + z31 = z14; + z17 = _mm512_ternarylogic_epi64(z17, z14, z6, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z6, z3, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z3, z20, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z9; + z31 = z1; + z9 = _mm512_ternarylogic_epi64(z9, z1, z23, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z23, z15, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z15, z12, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z21; + z31 = z18; + z21 = _mm512_ternarylogic_epi64(z21, z18, z10, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z10, z7, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z7, z4, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z13; + z31 = z5; + z13 = _mm512_ternarylogic_epi64(z13, z5, z2, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z2, z24, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z24, z16, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 128))); + /* Round 3 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z17, z9, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z21, z13, 0x96); + z26 = z22; + z26 = _mm512_ternarylogic_epi64(z26, z14, z1, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z18, z5, 0x96); + z27 = z19; + z27 = _mm512_ternarylogic_epi64(z27, z6, z23, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z10, z2, 0x96); + z28 = z11; + z28 = _mm512_ternarylogic_epi64(z28, z3, z15, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z7, z24, 0x96); + z29 = z8; + z29 = _mm512_ternarylogic_epi64(z29, z20, z12, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z4, z16, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z17 = _mm512_xor_si512(z17, z30); + z9 = _mm512_xor_si512(z9, z30); + z21 = _mm512_xor_si512(z21, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z22 = _mm512_xor_si512(z22, z30); + z14 = _mm512_xor_si512(z14, z30); + z1 = _mm512_xor_si512(z1, z30); + z18 = _mm512_xor_si512(z18, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z19 = _mm512_xor_si512(z19, z30); + z6 = _mm512_xor_si512(z6, z30); + z23 = _mm512_xor_si512(z23, z30); + z10 = _mm512_xor_si512(z10, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z11 = _mm512_xor_si512(z11, z30); + z3 = _mm512_xor_si512(z3, z30); + z15 = _mm512_xor_si512(z15, z30); + z7 = _mm512_xor_si512(z7, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z8 = _mm512_xor_si512(z8, z30); + z20 = _mm512_xor_si512(z20, z30); + z12 = _mm512_xor_si512(z12, z30); + z4 = _mm512_xor_si512(z4, z30); + z16 = _mm512_xor_si512(z16, z30); + /* Rho - rotate each lane in place */ + z22 = _mm512_rol_epi64(z22, 1); + z19 = _mm512_rol_epi64(z19, 62); + z11 = _mm512_rol_epi64(z11, 28); + z8 = _mm512_rol_epi64(z8, 27); + z17 = _mm512_rol_epi64(z17, 36); + z14 = _mm512_rol_epi64(z14, 44); + z6 = _mm512_rol_epi64(z6, 6); + z3 = _mm512_rol_epi64(z3, 55); + z20 = _mm512_rol_epi64(z20, 20); + z9 = _mm512_rol_epi64(z9, 3); + z1 = _mm512_rol_epi64(z1, 10); + z23 = _mm512_rol_epi64(z23, 43); + z15 = _mm512_rol_epi64(z15, 25); + z12 = _mm512_rol_epi64(z12, 39); + z21 = _mm512_rol_epi64(z21, 41); + z18 = _mm512_rol_epi64(z18, 45); + z10 = _mm512_rol_epi64(z10, 15); + z7 = _mm512_rol_epi64(z7, 21); + z4 = _mm512_rol_epi64(z4, 8); + z13 = _mm512_rol_epi64(z13, 18); + z5 = _mm512_rol_epi64(z5, 2); + z2 = _mm512_rol_epi64(z2, 61); + z24 = _mm512_rol_epi64(z24, 56); + z16 = _mm512_rol_epi64(z16, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z14; + z0 = _mm512_ternarylogic_epi64(z0, z14, z23, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z23, z7, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z7, z16, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z11; + z31 = z20; + z11 = _mm512_ternarylogic_epi64(z11, z20, z9, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z9, z18, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z18, z2, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z22; + z31 = z6; + z22 = _mm512_ternarylogic_epi64(z22, z6, z15, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z15, z4, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z4, z13, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z8; + z31 = z17; + z8 = _mm512_ternarylogic_epi64(z8, z17, z1, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z1, z10, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z10, z24, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z19; + z31 = z3; + z19 = _mm512_ternarylogic_epi64(z19, z3, z12, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z12, z21, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z21, z5, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 192))); + /* Round 4 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z11, z22, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z8, z19, 0x96); + z26 = z14; + z26 = _mm512_ternarylogic_epi64(z26, z20, z6, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z17, z3, 0x96); + z27 = z23; + z27 = _mm512_ternarylogic_epi64(z27, z9, z15, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z1, z12, 0x96); + z28 = z7; + z28 = _mm512_ternarylogic_epi64(z28, z18, z4, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z10, z21, 0x96); + z29 = z16; + z29 = _mm512_ternarylogic_epi64(z29, z2, z13, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z24, z5, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z11 = _mm512_xor_si512(z11, z30); + z22 = _mm512_xor_si512(z22, z30); + z8 = _mm512_xor_si512(z8, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z14 = _mm512_xor_si512(z14, z30); + z20 = _mm512_xor_si512(z20, z30); + z6 = _mm512_xor_si512(z6, z30); + z17 = _mm512_xor_si512(z17, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z23 = _mm512_xor_si512(z23, z30); + z9 = _mm512_xor_si512(z9, z30); + z15 = _mm512_xor_si512(z15, z30); + z1 = _mm512_xor_si512(z1, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z7 = _mm512_xor_si512(z7, z30); + z18 = _mm512_xor_si512(z18, z30); + z4 = _mm512_xor_si512(z4, z30); + z10 = _mm512_xor_si512(z10, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z16 = _mm512_xor_si512(z16, z30); + z2 = _mm512_xor_si512(z2, z30); + z13 = _mm512_xor_si512(z13, z30); + z24 = _mm512_xor_si512(z24, z30); + z5 = _mm512_xor_si512(z5, z30); + /* Rho - rotate each lane in place */ + z14 = _mm512_rol_epi64(z14, 1); + z23 = _mm512_rol_epi64(z23, 62); + z7 = _mm512_rol_epi64(z7, 28); + z16 = _mm512_rol_epi64(z16, 27); + z11 = _mm512_rol_epi64(z11, 36); + z20 = _mm512_rol_epi64(z20, 44); + z9 = _mm512_rol_epi64(z9, 6); + z18 = _mm512_rol_epi64(z18, 55); + z2 = _mm512_rol_epi64(z2, 20); + z22 = _mm512_rol_epi64(z22, 3); + z6 = _mm512_rol_epi64(z6, 10); + z15 = _mm512_rol_epi64(z15, 43); + z4 = _mm512_rol_epi64(z4, 25); + z13 = _mm512_rol_epi64(z13, 39); + z8 = _mm512_rol_epi64(z8, 41); + z17 = _mm512_rol_epi64(z17, 45); + z1 = _mm512_rol_epi64(z1, 15); + z10 = _mm512_rol_epi64(z10, 21); + z24 = _mm512_rol_epi64(z24, 8); + z19 = _mm512_rol_epi64(z19, 18); + z3 = _mm512_rol_epi64(z3, 2); + z12 = _mm512_rol_epi64(z12, 61); + z21 = _mm512_rol_epi64(z21, 56); + z5 = _mm512_rol_epi64(z5, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z20; + z0 = _mm512_ternarylogic_epi64(z0, z20, z15, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z15, z10, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z10, z5, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z7; + z31 = z2; + z7 = _mm512_ternarylogic_epi64(z7, z2, z22, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z22, z17, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z17, z12, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z14; + z31 = z9; + z14 = _mm512_ternarylogic_epi64(z14, z9, z4, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z4, z24, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z24, z19, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z16; + z31 = z11; + z16 = _mm512_ternarylogic_epi64(z16, z11, z6, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z6, z1, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z1, z21, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z23; + z31 = z18; + z23 = _mm512_ternarylogic_epi64(z23, z18, z13, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z13, z8, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z8, z3, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 256))); + /* Round 5 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z7, z14, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z16, z23, 0x96); + z26 = z20; + z26 = _mm512_ternarylogic_epi64(z26, z2, z9, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z11, z18, 0x96); + z27 = z15; + z27 = _mm512_ternarylogic_epi64(z27, z22, z4, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z6, z13, 0x96); + z28 = z10; + z28 = _mm512_ternarylogic_epi64(z28, z17, z24, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z1, z8, 0x96); + z29 = z5; + z29 = _mm512_ternarylogic_epi64(z29, z12, z19, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z21, z3, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z7 = _mm512_xor_si512(z7, z30); + z14 = _mm512_xor_si512(z14, z30); + z16 = _mm512_xor_si512(z16, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z20 = _mm512_xor_si512(z20, z30); + z2 = _mm512_xor_si512(z2, z30); + z9 = _mm512_xor_si512(z9, z30); + z11 = _mm512_xor_si512(z11, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z15 = _mm512_xor_si512(z15, z30); + z22 = _mm512_xor_si512(z22, z30); + z4 = _mm512_xor_si512(z4, z30); + z6 = _mm512_xor_si512(z6, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z10 = _mm512_xor_si512(z10, z30); + z17 = _mm512_xor_si512(z17, z30); + z24 = _mm512_xor_si512(z24, z30); + z1 = _mm512_xor_si512(z1, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z5 = _mm512_xor_si512(z5, z30); + z12 = _mm512_xor_si512(z12, z30); + z19 = _mm512_xor_si512(z19, z30); + z21 = _mm512_xor_si512(z21, z30); + z3 = _mm512_xor_si512(z3, z30); + /* Rho - rotate each lane in place */ + z20 = _mm512_rol_epi64(z20, 1); + z15 = _mm512_rol_epi64(z15, 62); + z10 = _mm512_rol_epi64(z10, 28); + z5 = _mm512_rol_epi64(z5, 27); + z7 = _mm512_rol_epi64(z7, 36); + z2 = _mm512_rol_epi64(z2, 44); + z22 = _mm512_rol_epi64(z22, 6); + z17 = _mm512_rol_epi64(z17, 55); + z12 = _mm512_rol_epi64(z12, 20); + z14 = _mm512_rol_epi64(z14, 3); + z9 = _mm512_rol_epi64(z9, 10); + z4 = _mm512_rol_epi64(z4, 43); + z24 = _mm512_rol_epi64(z24, 25); + z19 = _mm512_rol_epi64(z19, 39); + z16 = _mm512_rol_epi64(z16, 41); + z11 = _mm512_rol_epi64(z11, 45); + z6 = _mm512_rol_epi64(z6, 15); + z1 = _mm512_rol_epi64(z1, 21); + z21 = _mm512_rol_epi64(z21, 8); + z23 = _mm512_rol_epi64(z23, 18); + z18 = _mm512_rol_epi64(z18, 2); + z13 = _mm512_rol_epi64(z13, 61); + z8 = _mm512_rol_epi64(z8, 56); + z3 = _mm512_rol_epi64(z3, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z2; + z0 = _mm512_ternarylogic_epi64(z0, z2, z4, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z4, z1, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z1, z3, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z10; + z31 = z12; + z10 = _mm512_ternarylogic_epi64(z10, z12, z14, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z14, z11, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z11, z13, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z20; + z31 = z22; + z20 = _mm512_ternarylogic_epi64(z20, z22, z24, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z24, z21, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z21, z23, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z5; + z31 = z7; + z5 = _mm512_ternarylogic_epi64(z5, z7, z9, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z9, z6, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z6, z8, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z15; + z31 = z17; + z15 = _mm512_ternarylogic_epi64(z15, z17, z19, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z19, z16, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z16, z18, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 320))); + /* Round 6 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z10, z20, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z5, z15, 0x96); + z26 = z2; + z26 = _mm512_ternarylogic_epi64(z26, z12, z22, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z7, z17, 0x96); + z27 = z4; + z27 = _mm512_ternarylogic_epi64(z27, z14, z24, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z9, z19, 0x96); + z28 = z1; + z28 = _mm512_ternarylogic_epi64(z28, z11, z21, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z6, z16, 0x96); + z29 = z3; + z29 = _mm512_ternarylogic_epi64(z29, z13, z23, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z8, z18, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z10 = _mm512_xor_si512(z10, z30); + z20 = _mm512_xor_si512(z20, z30); + z5 = _mm512_xor_si512(z5, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z2 = _mm512_xor_si512(z2, z30); + z12 = _mm512_xor_si512(z12, z30); + z22 = _mm512_xor_si512(z22, z30); + z7 = _mm512_xor_si512(z7, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z4 = _mm512_xor_si512(z4, z30); + z14 = _mm512_xor_si512(z14, z30); + z24 = _mm512_xor_si512(z24, z30); + z9 = _mm512_xor_si512(z9, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z1 = _mm512_xor_si512(z1, z30); + z11 = _mm512_xor_si512(z11, z30); + z21 = _mm512_xor_si512(z21, z30); + z6 = _mm512_xor_si512(z6, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z3 = _mm512_xor_si512(z3, z30); + z13 = _mm512_xor_si512(z13, z30); + z23 = _mm512_xor_si512(z23, z30); + z8 = _mm512_xor_si512(z8, z30); + z18 = _mm512_xor_si512(z18, z30); + /* Rho - rotate each lane in place */ + z2 = _mm512_rol_epi64(z2, 1); + z4 = _mm512_rol_epi64(z4, 62); + z1 = _mm512_rol_epi64(z1, 28); + z3 = _mm512_rol_epi64(z3, 27); + z10 = _mm512_rol_epi64(z10, 36); + z12 = _mm512_rol_epi64(z12, 44); + z14 = _mm512_rol_epi64(z14, 6); + z11 = _mm512_rol_epi64(z11, 55); + z13 = _mm512_rol_epi64(z13, 20); + z20 = _mm512_rol_epi64(z20, 3); + z22 = _mm512_rol_epi64(z22, 10); + z24 = _mm512_rol_epi64(z24, 43); + z21 = _mm512_rol_epi64(z21, 25); + z23 = _mm512_rol_epi64(z23, 39); + z5 = _mm512_rol_epi64(z5, 41); + z7 = _mm512_rol_epi64(z7, 45); + z9 = _mm512_rol_epi64(z9, 15); + z6 = _mm512_rol_epi64(z6, 21); + z8 = _mm512_rol_epi64(z8, 8); + z15 = _mm512_rol_epi64(z15, 18); + z17 = _mm512_rol_epi64(z17, 2); + z19 = _mm512_rol_epi64(z19, 61); + z16 = _mm512_rol_epi64(z16, 56); + z18 = _mm512_rol_epi64(z18, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z12; + z0 = _mm512_ternarylogic_epi64(z0, z12, z24, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z24, z6, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z6, z18, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z1; + z31 = z13; + z1 = _mm512_ternarylogic_epi64(z1, z13, z20, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z20, z7, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z7, z19, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z2; + z31 = z14; + z2 = _mm512_ternarylogic_epi64(z2, z14, z21, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z21, z8, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z8, z15, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z3; + z31 = z10; + z3 = _mm512_ternarylogic_epi64(z3, z10, z22, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z22, z9, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z9, z16, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z4; + z31 = z11; + z4 = _mm512_ternarylogic_epi64(z4, z11, z23, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z23, z5, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z5, z17, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 384))); + /* Round 7 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z1, z2, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z3, z4, 0x96); + z26 = z12; + z26 = _mm512_ternarylogic_epi64(z26, z13, z14, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z10, z11, 0x96); + z27 = z24; + z27 = _mm512_ternarylogic_epi64(z27, z20, z21, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z22, z23, 0x96); + z28 = z6; + z28 = _mm512_ternarylogic_epi64(z28, z7, z8, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z9, z5, 0x96); + z29 = z18; + z29 = _mm512_ternarylogic_epi64(z29, z19, z15, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z16, z17, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z1 = _mm512_xor_si512(z1, z30); + z2 = _mm512_xor_si512(z2, z30); + z3 = _mm512_xor_si512(z3, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z12 = _mm512_xor_si512(z12, z30); + z13 = _mm512_xor_si512(z13, z30); + z14 = _mm512_xor_si512(z14, z30); + z10 = _mm512_xor_si512(z10, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z24 = _mm512_xor_si512(z24, z30); + z20 = _mm512_xor_si512(z20, z30); + z21 = _mm512_xor_si512(z21, z30); + z22 = _mm512_xor_si512(z22, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z6 = _mm512_xor_si512(z6, z30); + z7 = _mm512_xor_si512(z7, z30); + z8 = _mm512_xor_si512(z8, z30); + z9 = _mm512_xor_si512(z9, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z18 = _mm512_xor_si512(z18, z30); + z19 = _mm512_xor_si512(z19, z30); + z15 = _mm512_xor_si512(z15, z30); + z16 = _mm512_xor_si512(z16, z30); + z17 = _mm512_xor_si512(z17, z30); + /* Rho - rotate each lane in place */ + z12 = _mm512_rol_epi64(z12, 1); + z24 = _mm512_rol_epi64(z24, 62); + z6 = _mm512_rol_epi64(z6, 28); + z18 = _mm512_rol_epi64(z18, 27); + z1 = _mm512_rol_epi64(z1, 36); + z13 = _mm512_rol_epi64(z13, 44); + z20 = _mm512_rol_epi64(z20, 6); + z7 = _mm512_rol_epi64(z7, 55); + z19 = _mm512_rol_epi64(z19, 20); + z2 = _mm512_rol_epi64(z2, 3); + z14 = _mm512_rol_epi64(z14, 10); + z21 = _mm512_rol_epi64(z21, 43); + z8 = _mm512_rol_epi64(z8, 25); + z15 = _mm512_rol_epi64(z15, 39); + z3 = _mm512_rol_epi64(z3, 41); + z10 = _mm512_rol_epi64(z10, 45); + z22 = _mm512_rol_epi64(z22, 15); + z9 = _mm512_rol_epi64(z9, 21); + z16 = _mm512_rol_epi64(z16, 8); + z4 = _mm512_rol_epi64(z4, 18); + z11 = _mm512_rol_epi64(z11, 2); + z23 = _mm512_rol_epi64(z23, 61); + z5 = _mm512_rol_epi64(z5, 56); + z17 = _mm512_rol_epi64(z17, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z13; + z0 = _mm512_ternarylogic_epi64(z0, z13, z21, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z21, z9, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z9, z17, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z6; + z31 = z19; + z6 = _mm512_ternarylogic_epi64(z6, z19, z2, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z2, z10, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z10, z23, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z12; + z31 = z20; + z12 = _mm512_ternarylogic_epi64(z12, z20, z8, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z8, z16, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z16, z4, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z18; + z31 = z1; + z18 = _mm512_ternarylogic_epi64(z18, z1, z14, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z14, z22, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z22, z5, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z24; + z31 = z7; + z24 = _mm512_ternarylogic_epi64(z24, z7, z15, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z15, z3, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z3, z11, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 448))); + /* Round 8 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z6, z12, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z18, z24, 0x96); + z26 = z13; + z26 = _mm512_ternarylogic_epi64(z26, z19, z20, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z1, z7, 0x96); + z27 = z21; + z27 = _mm512_ternarylogic_epi64(z27, z2, z8, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z14, z15, 0x96); + z28 = z9; + z28 = _mm512_ternarylogic_epi64(z28, z10, z16, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z22, z3, 0x96); + z29 = z17; + z29 = _mm512_ternarylogic_epi64(z29, z23, z4, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z5, z11, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z6 = _mm512_xor_si512(z6, z30); + z12 = _mm512_xor_si512(z12, z30); + z18 = _mm512_xor_si512(z18, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z13 = _mm512_xor_si512(z13, z30); + z19 = _mm512_xor_si512(z19, z30); + z20 = _mm512_xor_si512(z20, z30); + z1 = _mm512_xor_si512(z1, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z21 = _mm512_xor_si512(z21, z30); + z2 = _mm512_xor_si512(z2, z30); + z8 = _mm512_xor_si512(z8, z30); + z14 = _mm512_xor_si512(z14, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z9 = _mm512_xor_si512(z9, z30); + z10 = _mm512_xor_si512(z10, z30); + z16 = _mm512_xor_si512(z16, z30); + z22 = _mm512_xor_si512(z22, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z17 = _mm512_xor_si512(z17, z30); + z23 = _mm512_xor_si512(z23, z30); + z4 = _mm512_xor_si512(z4, z30); + z5 = _mm512_xor_si512(z5, z30); + z11 = _mm512_xor_si512(z11, z30); + /* Rho - rotate each lane in place */ + z13 = _mm512_rol_epi64(z13, 1); + z21 = _mm512_rol_epi64(z21, 62); + z9 = _mm512_rol_epi64(z9, 28); + z17 = _mm512_rol_epi64(z17, 27); + z6 = _mm512_rol_epi64(z6, 36); + z19 = _mm512_rol_epi64(z19, 44); + z2 = _mm512_rol_epi64(z2, 6); + z10 = _mm512_rol_epi64(z10, 55); + z23 = _mm512_rol_epi64(z23, 20); + z12 = _mm512_rol_epi64(z12, 3); + z20 = _mm512_rol_epi64(z20, 10); + z8 = _mm512_rol_epi64(z8, 43); + z16 = _mm512_rol_epi64(z16, 25); + z4 = _mm512_rol_epi64(z4, 39); + z18 = _mm512_rol_epi64(z18, 41); + z1 = _mm512_rol_epi64(z1, 45); + z14 = _mm512_rol_epi64(z14, 15); + z22 = _mm512_rol_epi64(z22, 21); + z5 = _mm512_rol_epi64(z5, 8); + z24 = _mm512_rol_epi64(z24, 18); + z7 = _mm512_rol_epi64(z7, 2); + z15 = _mm512_rol_epi64(z15, 61); + z3 = _mm512_rol_epi64(z3, 56); + z11 = _mm512_rol_epi64(z11, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z19; + z0 = _mm512_ternarylogic_epi64(z0, z19, z8, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z8, z22, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z22, z11, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z9; + z31 = z23; + z9 = _mm512_ternarylogic_epi64(z9, z23, z12, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z12, z1, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z1, z15, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z13; + z31 = z2; + z13 = _mm512_ternarylogic_epi64(z13, z2, z16, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z16, z5, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z5, z24, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z17; + z31 = z6; + z17 = _mm512_ternarylogic_epi64(z17, z6, z20, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z20, z14, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z14, z3, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z21; + z31 = z10; + z21 = _mm512_ternarylogic_epi64(z21, z10, z4, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z4, z18, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z18, z7, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 512))); + /* Round 9 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z9, z13, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z17, z21, 0x96); + z26 = z19; + z26 = _mm512_ternarylogic_epi64(z26, z23, z2, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z6, z10, 0x96); + z27 = z8; + z27 = _mm512_ternarylogic_epi64(z27, z12, z16, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z20, z4, 0x96); + z28 = z22; + z28 = _mm512_ternarylogic_epi64(z28, z1, z5, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z14, z18, 0x96); + z29 = z11; + z29 = _mm512_ternarylogic_epi64(z29, z15, z24, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z3, z7, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z9 = _mm512_xor_si512(z9, z30); + z13 = _mm512_xor_si512(z13, z30); + z17 = _mm512_xor_si512(z17, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z19 = _mm512_xor_si512(z19, z30); + z23 = _mm512_xor_si512(z23, z30); + z2 = _mm512_xor_si512(z2, z30); + z6 = _mm512_xor_si512(z6, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z8 = _mm512_xor_si512(z8, z30); + z12 = _mm512_xor_si512(z12, z30); + z16 = _mm512_xor_si512(z16, z30); + z20 = _mm512_xor_si512(z20, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z22 = _mm512_xor_si512(z22, z30); + z1 = _mm512_xor_si512(z1, z30); + z5 = _mm512_xor_si512(z5, z30); + z14 = _mm512_xor_si512(z14, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z11 = _mm512_xor_si512(z11, z30); + z15 = _mm512_xor_si512(z15, z30); + z24 = _mm512_xor_si512(z24, z30); + z3 = _mm512_xor_si512(z3, z30); + z7 = _mm512_xor_si512(z7, z30); + /* Rho - rotate each lane in place */ + z19 = _mm512_rol_epi64(z19, 1); + z8 = _mm512_rol_epi64(z8, 62); + z22 = _mm512_rol_epi64(z22, 28); + z11 = _mm512_rol_epi64(z11, 27); + z9 = _mm512_rol_epi64(z9, 36); + z23 = _mm512_rol_epi64(z23, 44); + z12 = _mm512_rol_epi64(z12, 6); + z1 = _mm512_rol_epi64(z1, 55); + z15 = _mm512_rol_epi64(z15, 20); + z13 = _mm512_rol_epi64(z13, 3); + z2 = _mm512_rol_epi64(z2, 10); + z16 = _mm512_rol_epi64(z16, 43); + z5 = _mm512_rol_epi64(z5, 25); + z24 = _mm512_rol_epi64(z24, 39); + z17 = _mm512_rol_epi64(z17, 41); + z6 = _mm512_rol_epi64(z6, 45); + z20 = _mm512_rol_epi64(z20, 15); + z14 = _mm512_rol_epi64(z14, 21); + z3 = _mm512_rol_epi64(z3, 8); + z21 = _mm512_rol_epi64(z21, 18); + z10 = _mm512_rol_epi64(z10, 2); + z4 = _mm512_rol_epi64(z4, 61); + z18 = _mm512_rol_epi64(z18, 56); + z7 = _mm512_rol_epi64(z7, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z23; + z0 = _mm512_ternarylogic_epi64(z0, z23, z16, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z16, z14, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z14, z7, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z22; + z31 = z15; + z22 = _mm512_ternarylogic_epi64(z22, z15, z13, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z13, z6, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z6, z4, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z19; + z31 = z12; + z19 = _mm512_ternarylogic_epi64(z19, z12, z5, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z5, z3, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z3, z21, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z11; + z31 = z9; + z11 = _mm512_ternarylogic_epi64(z11, z9, z2, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z2, z20, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z20, z18, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z8; + z31 = z1; + z8 = _mm512_ternarylogic_epi64(z8, z1, z24, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z24, z17, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z17, z10, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 576))); + /* Round 10 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z22, z19, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z11, z8, 0x96); + z26 = z23; + z26 = _mm512_ternarylogic_epi64(z26, z15, z12, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z9, z1, 0x96); + z27 = z16; + z27 = _mm512_ternarylogic_epi64(z27, z13, z5, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z2, z24, 0x96); + z28 = z14; + z28 = _mm512_ternarylogic_epi64(z28, z6, z3, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z20, z17, 0x96); + z29 = z7; + z29 = _mm512_ternarylogic_epi64(z29, z4, z21, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z18, z10, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z22 = _mm512_xor_si512(z22, z30); + z19 = _mm512_xor_si512(z19, z30); + z11 = _mm512_xor_si512(z11, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z23 = _mm512_xor_si512(z23, z30); + z15 = _mm512_xor_si512(z15, z30); + z12 = _mm512_xor_si512(z12, z30); + z9 = _mm512_xor_si512(z9, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z16 = _mm512_xor_si512(z16, z30); + z13 = _mm512_xor_si512(z13, z30); + z5 = _mm512_xor_si512(z5, z30); + z2 = _mm512_xor_si512(z2, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z14 = _mm512_xor_si512(z14, z30); + z6 = _mm512_xor_si512(z6, z30); + z3 = _mm512_xor_si512(z3, z30); + z20 = _mm512_xor_si512(z20, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z7 = _mm512_xor_si512(z7, z30); + z4 = _mm512_xor_si512(z4, z30); + z21 = _mm512_xor_si512(z21, z30); + z18 = _mm512_xor_si512(z18, z30); + z10 = _mm512_xor_si512(z10, z30); + /* Rho - rotate each lane in place */ + z23 = _mm512_rol_epi64(z23, 1); + z16 = _mm512_rol_epi64(z16, 62); + z14 = _mm512_rol_epi64(z14, 28); + z7 = _mm512_rol_epi64(z7, 27); + z22 = _mm512_rol_epi64(z22, 36); + z15 = _mm512_rol_epi64(z15, 44); + z13 = _mm512_rol_epi64(z13, 6); + z6 = _mm512_rol_epi64(z6, 55); + z4 = _mm512_rol_epi64(z4, 20); + z19 = _mm512_rol_epi64(z19, 3); + z12 = _mm512_rol_epi64(z12, 10); + z5 = _mm512_rol_epi64(z5, 43); + z3 = _mm512_rol_epi64(z3, 25); + z21 = _mm512_rol_epi64(z21, 39); + z11 = _mm512_rol_epi64(z11, 41); + z9 = _mm512_rol_epi64(z9, 45); + z2 = _mm512_rol_epi64(z2, 15); + z20 = _mm512_rol_epi64(z20, 21); + z18 = _mm512_rol_epi64(z18, 8); + z8 = _mm512_rol_epi64(z8, 18); + z1 = _mm512_rol_epi64(z1, 2); + z24 = _mm512_rol_epi64(z24, 61); + z17 = _mm512_rol_epi64(z17, 56); + z10 = _mm512_rol_epi64(z10, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z15; + z0 = _mm512_ternarylogic_epi64(z0, z15, z5, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z5, z20, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z20, z10, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z14; + z31 = z4; + z14 = _mm512_ternarylogic_epi64(z14, z4, z19, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z19, z9, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z9, z24, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z23; + z31 = z13; + z23 = _mm512_ternarylogic_epi64(z23, z13, z3, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z3, z18, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z18, z8, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z7; + z31 = z22; + z7 = _mm512_ternarylogic_epi64(z7, z22, z12, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z12, z2, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z2, z17, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z16; + z31 = z6; + z16 = _mm512_ternarylogic_epi64(z16, z6, z21, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z21, z11, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z11, z1, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 640))); + /* Round 11 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z14, z23, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z7, z16, 0x96); + z26 = z15; + z26 = _mm512_ternarylogic_epi64(z26, z4, z13, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z22, z6, 0x96); + z27 = z5; + z27 = _mm512_ternarylogic_epi64(z27, z19, z3, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z12, z21, 0x96); + z28 = z20; + z28 = _mm512_ternarylogic_epi64(z28, z9, z18, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z2, z11, 0x96); + z29 = z10; + z29 = _mm512_ternarylogic_epi64(z29, z24, z8, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z17, z1, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z14 = _mm512_xor_si512(z14, z30); + z23 = _mm512_xor_si512(z23, z30); + z7 = _mm512_xor_si512(z7, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z15 = _mm512_xor_si512(z15, z30); + z4 = _mm512_xor_si512(z4, z30); + z13 = _mm512_xor_si512(z13, z30); + z22 = _mm512_xor_si512(z22, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z5 = _mm512_xor_si512(z5, z30); + z19 = _mm512_xor_si512(z19, z30); + z3 = _mm512_xor_si512(z3, z30); + z12 = _mm512_xor_si512(z12, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z20 = _mm512_xor_si512(z20, z30); + z9 = _mm512_xor_si512(z9, z30); + z18 = _mm512_xor_si512(z18, z30); + z2 = _mm512_xor_si512(z2, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z10 = _mm512_xor_si512(z10, z30); + z24 = _mm512_xor_si512(z24, z30); + z8 = _mm512_xor_si512(z8, z30); + z17 = _mm512_xor_si512(z17, z30); + z1 = _mm512_xor_si512(z1, z30); + /* Rho - rotate each lane in place */ + z15 = _mm512_rol_epi64(z15, 1); + z5 = _mm512_rol_epi64(z5, 62); + z20 = _mm512_rol_epi64(z20, 28); + z10 = _mm512_rol_epi64(z10, 27); + z14 = _mm512_rol_epi64(z14, 36); + z4 = _mm512_rol_epi64(z4, 44); + z19 = _mm512_rol_epi64(z19, 6); + z9 = _mm512_rol_epi64(z9, 55); + z24 = _mm512_rol_epi64(z24, 20); + z23 = _mm512_rol_epi64(z23, 3); + z13 = _mm512_rol_epi64(z13, 10); + z3 = _mm512_rol_epi64(z3, 43); + z18 = _mm512_rol_epi64(z18, 25); + z8 = _mm512_rol_epi64(z8, 39); + z7 = _mm512_rol_epi64(z7, 41); + z22 = _mm512_rol_epi64(z22, 45); + z12 = _mm512_rol_epi64(z12, 15); + z2 = _mm512_rol_epi64(z2, 21); + z17 = _mm512_rol_epi64(z17, 8); + z16 = _mm512_rol_epi64(z16, 18); + z6 = _mm512_rol_epi64(z6, 2); + z21 = _mm512_rol_epi64(z21, 61); + z11 = _mm512_rol_epi64(z11, 56); + z1 = _mm512_rol_epi64(z1, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z4; + z0 = _mm512_ternarylogic_epi64(z0, z4, z3, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z3, z2, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z2, z1, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z20; + z31 = z24; + z20 = _mm512_ternarylogic_epi64(z20, z24, z23, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z23, z22, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z22, z21, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z15; + z31 = z19; + z15 = _mm512_ternarylogic_epi64(z15, z19, z18, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z18, z17, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z17, z16, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z10; + z31 = z14; + z10 = _mm512_ternarylogic_epi64(z10, z14, z13, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z13, z12, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z12, z11, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z5; + z31 = z9; + z5 = _mm512_ternarylogic_epi64(z5, z9, z8, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z8, z7, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z7, z6, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 704))); + /* Round 12 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z20, z15, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z10, z5, 0x96); + z26 = z4; + z26 = _mm512_ternarylogic_epi64(z26, z24, z19, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z14, z9, 0x96); + z27 = z3; + z27 = _mm512_ternarylogic_epi64(z27, z23, z18, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z13, z8, 0x96); + z28 = z2; + z28 = _mm512_ternarylogic_epi64(z28, z22, z17, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z12, z7, 0x96); + z29 = z1; + z29 = _mm512_ternarylogic_epi64(z29, z21, z16, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z11, z6, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z20 = _mm512_xor_si512(z20, z30); + z15 = _mm512_xor_si512(z15, z30); + z10 = _mm512_xor_si512(z10, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z4 = _mm512_xor_si512(z4, z30); + z24 = _mm512_xor_si512(z24, z30); + z19 = _mm512_xor_si512(z19, z30); + z14 = _mm512_xor_si512(z14, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z3 = _mm512_xor_si512(z3, z30); + z23 = _mm512_xor_si512(z23, z30); + z18 = _mm512_xor_si512(z18, z30); + z13 = _mm512_xor_si512(z13, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z2 = _mm512_xor_si512(z2, z30); + z22 = _mm512_xor_si512(z22, z30); + z17 = _mm512_xor_si512(z17, z30); + z12 = _mm512_xor_si512(z12, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z1 = _mm512_xor_si512(z1, z30); + z21 = _mm512_xor_si512(z21, z30); + z16 = _mm512_xor_si512(z16, z30); + z11 = _mm512_xor_si512(z11, z30); + z6 = _mm512_xor_si512(z6, z30); + /* Rho - rotate each lane in place */ + z4 = _mm512_rol_epi64(z4, 1); + z3 = _mm512_rol_epi64(z3, 62); + z2 = _mm512_rol_epi64(z2, 28); + z1 = _mm512_rol_epi64(z1, 27); + z20 = _mm512_rol_epi64(z20, 36); + z24 = _mm512_rol_epi64(z24, 44); + z23 = _mm512_rol_epi64(z23, 6); + z22 = _mm512_rol_epi64(z22, 55); + z21 = _mm512_rol_epi64(z21, 20); + z15 = _mm512_rol_epi64(z15, 3); + z19 = _mm512_rol_epi64(z19, 10); + z18 = _mm512_rol_epi64(z18, 43); + z17 = _mm512_rol_epi64(z17, 25); + z16 = _mm512_rol_epi64(z16, 39); + z10 = _mm512_rol_epi64(z10, 41); + z14 = _mm512_rol_epi64(z14, 45); + z13 = _mm512_rol_epi64(z13, 15); + z12 = _mm512_rol_epi64(z12, 21); + z11 = _mm512_rol_epi64(z11, 8); + z5 = _mm512_rol_epi64(z5, 18); + z9 = _mm512_rol_epi64(z9, 2); + z8 = _mm512_rol_epi64(z8, 61); + z7 = _mm512_rol_epi64(z7, 56); + z6 = _mm512_rol_epi64(z6, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z24; + z0 = _mm512_ternarylogic_epi64(z0, z24, z18, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z18, z12, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z12, z6, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z2; + z31 = z21; + z2 = _mm512_ternarylogic_epi64(z2, z21, z15, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z15, z14, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z14, z8, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z4; + z31 = z23; + z4 = _mm512_ternarylogic_epi64(z4, z23, z17, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z17, z11, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z11, z5, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z1; + z31 = z20; + z1 = _mm512_ternarylogic_epi64(z1, z20, z19, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z19, z13, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z13, z7, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z3; + z31 = z22; + z3 = _mm512_ternarylogic_epi64(z3, z22, z16, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z16, z10, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z10, z9, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 768))); + /* Round 13 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z2, z4, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z1, z3, 0x96); + z26 = z24; + z26 = _mm512_ternarylogic_epi64(z26, z21, z23, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z20, z22, 0x96); + z27 = z18; + z27 = _mm512_ternarylogic_epi64(z27, z15, z17, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z19, z16, 0x96); + z28 = z12; + z28 = _mm512_ternarylogic_epi64(z28, z14, z11, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z13, z10, 0x96); + z29 = z6; + z29 = _mm512_ternarylogic_epi64(z29, z8, z5, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z7, z9, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z2 = _mm512_xor_si512(z2, z30); + z4 = _mm512_xor_si512(z4, z30); + z1 = _mm512_xor_si512(z1, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z24 = _mm512_xor_si512(z24, z30); + z21 = _mm512_xor_si512(z21, z30); + z23 = _mm512_xor_si512(z23, z30); + z20 = _mm512_xor_si512(z20, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z18 = _mm512_xor_si512(z18, z30); + z15 = _mm512_xor_si512(z15, z30); + z17 = _mm512_xor_si512(z17, z30); + z19 = _mm512_xor_si512(z19, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z12 = _mm512_xor_si512(z12, z30); + z14 = _mm512_xor_si512(z14, z30); + z11 = _mm512_xor_si512(z11, z30); + z13 = _mm512_xor_si512(z13, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z6 = _mm512_xor_si512(z6, z30); + z8 = _mm512_xor_si512(z8, z30); + z5 = _mm512_xor_si512(z5, z30); + z7 = _mm512_xor_si512(z7, z30); + z9 = _mm512_xor_si512(z9, z30); + /* Rho - rotate each lane in place */ + z24 = _mm512_rol_epi64(z24, 1); + z18 = _mm512_rol_epi64(z18, 62); + z12 = _mm512_rol_epi64(z12, 28); + z6 = _mm512_rol_epi64(z6, 27); + z2 = _mm512_rol_epi64(z2, 36); + z21 = _mm512_rol_epi64(z21, 44); + z15 = _mm512_rol_epi64(z15, 6); + z14 = _mm512_rol_epi64(z14, 55); + z8 = _mm512_rol_epi64(z8, 20); + z4 = _mm512_rol_epi64(z4, 3); + z23 = _mm512_rol_epi64(z23, 10); + z17 = _mm512_rol_epi64(z17, 43); + z11 = _mm512_rol_epi64(z11, 25); + z5 = _mm512_rol_epi64(z5, 39); + z1 = _mm512_rol_epi64(z1, 41); + z20 = _mm512_rol_epi64(z20, 45); + z19 = _mm512_rol_epi64(z19, 15); + z13 = _mm512_rol_epi64(z13, 21); + z7 = _mm512_rol_epi64(z7, 8); + z3 = _mm512_rol_epi64(z3, 18); + z22 = _mm512_rol_epi64(z22, 2); + z16 = _mm512_rol_epi64(z16, 61); + z10 = _mm512_rol_epi64(z10, 56); + z9 = _mm512_rol_epi64(z9, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z21; + z0 = _mm512_ternarylogic_epi64(z0, z21, z17, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z17, z13, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z13, z9, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z12; + z31 = z8; + z12 = _mm512_ternarylogic_epi64(z12, z8, z4, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z4, z20, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z20, z16, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z24; + z31 = z15; + z24 = _mm512_ternarylogic_epi64(z24, z15, z11, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z11, z7, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z7, z3, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z6; + z31 = z2; + z6 = _mm512_ternarylogic_epi64(z6, z2, z23, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z23, z19, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z19, z10, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z18; + z31 = z14; + z18 = _mm512_ternarylogic_epi64(z18, z14, z5, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z5, z1, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z1, z22, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 832))); + /* Round 14 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z12, z24, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z6, z18, 0x96); + z26 = z21; + z26 = _mm512_ternarylogic_epi64(z26, z8, z15, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z2, z14, 0x96); + z27 = z17; + z27 = _mm512_ternarylogic_epi64(z27, z4, z11, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z23, z5, 0x96); + z28 = z13; + z28 = _mm512_ternarylogic_epi64(z28, z20, z7, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z19, z1, 0x96); + z29 = z9; + z29 = _mm512_ternarylogic_epi64(z29, z16, z3, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z10, z22, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z12 = _mm512_xor_si512(z12, z30); + z24 = _mm512_xor_si512(z24, z30); + z6 = _mm512_xor_si512(z6, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z21 = _mm512_xor_si512(z21, z30); + z8 = _mm512_xor_si512(z8, z30); + z15 = _mm512_xor_si512(z15, z30); + z2 = _mm512_xor_si512(z2, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z17 = _mm512_xor_si512(z17, z30); + z4 = _mm512_xor_si512(z4, z30); + z11 = _mm512_xor_si512(z11, z30); + z23 = _mm512_xor_si512(z23, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z13 = _mm512_xor_si512(z13, z30); + z20 = _mm512_xor_si512(z20, z30); + z7 = _mm512_xor_si512(z7, z30); + z19 = _mm512_xor_si512(z19, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z9 = _mm512_xor_si512(z9, z30); + z16 = _mm512_xor_si512(z16, z30); + z3 = _mm512_xor_si512(z3, z30); + z10 = _mm512_xor_si512(z10, z30); + z22 = _mm512_xor_si512(z22, z30); + /* Rho - rotate each lane in place */ + z21 = _mm512_rol_epi64(z21, 1); + z17 = _mm512_rol_epi64(z17, 62); + z13 = _mm512_rol_epi64(z13, 28); + z9 = _mm512_rol_epi64(z9, 27); + z12 = _mm512_rol_epi64(z12, 36); + z8 = _mm512_rol_epi64(z8, 44); + z4 = _mm512_rol_epi64(z4, 6); + z20 = _mm512_rol_epi64(z20, 55); + z16 = _mm512_rol_epi64(z16, 20); + z24 = _mm512_rol_epi64(z24, 3); + z15 = _mm512_rol_epi64(z15, 10); + z11 = _mm512_rol_epi64(z11, 43); + z7 = _mm512_rol_epi64(z7, 25); + z3 = _mm512_rol_epi64(z3, 39); + z6 = _mm512_rol_epi64(z6, 41); + z2 = _mm512_rol_epi64(z2, 45); + z23 = _mm512_rol_epi64(z23, 15); + z19 = _mm512_rol_epi64(z19, 21); + z10 = _mm512_rol_epi64(z10, 8); + z18 = _mm512_rol_epi64(z18, 18); + z14 = _mm512_rol_epi64(z14, 2); + z5 = _mm512_rol_epi64(z5, 61); + z1 = _mm512_rol_epi64(z1, 56); + z22 = _mm512_rol_epi64(z22, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z8; + z0 = _mm512_ternarylogic_epi64(z0, z8, z11, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z11, z19, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z19, z22, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z13; + z31 = z16; + z13 = _mm512_ternarylogic_epi64(z13, z16, z24, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z24, z2, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z2, z5, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z21; + z31 = z4; + z21 = _mm512_ternarylogic_epi64(z21, z4, z7, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z7, z10, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z10, z18, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z9; + z31 = z12; + z9 = _mm512_ternarylogic_epi64(z9, z12, z15, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z15, z23, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z23, z1, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z17; + z31 = z20; + z17 = _mm512_ternarylogic_epi64(z17, z20, z3, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z3, z6, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z6, z14, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 896))); + /* Round 15 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z13, z21, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z9, z17, 0x96); + z26 = z8; + z26 = _mm512_ternarylogic_epi64(z26, z16, z4, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z12, z20, 0x96); + z27 = z11; + z27 = _mm512_ternarylogic_epi64(z27, z24, z7, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z15, z3, 0x96); + z28 = z19; + z28 = _mm512_ternarylogic_epi64(z28, z2, z10, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z23, z6, 0x96); + z29 = z22; + z29 = _mm512_ternarylogic_epi64(z29, z5, z18, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z1, z14, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z13 = _mm512_xor_si512(z13, z30); + z21 = _mm512_xor_si512(z21, z30); + z9 = _mm512_xor_si512(z9, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z8 = _mm512_xor_si512(z8, z30); + z16 = _mm512_xor_si512(z16, z30); + z4 = _mm512_xor_si512(z4, z30); + z12 = _mm512_xor_si512(z12, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z11 = _mm512_xor_si512(z11, z30); + z24 = _mm512_xor_si512(z24, z30); + z7 = _mm512_xor_si512(z7, z30); + z15 = _mm512_xor_si512(z15, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z19 = _mm512_xor_si512(z19, z30); + z2 = _mm512_xor_si512(z2, z30); + z10 = _mm512_xor_si512(z10, z30); + z23 = _mm512_xor_si512(z23, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z22 = _mm512_xor_si512(z22, z30); + z5 = _mm512_xor_si512(z5, z30); + z18 = _mm512_xor_si512(z18, z30); + z1 = _mm512_xor_si512(z1, z30); + z14 = _mm512_xor_si512(z14, z30); + /* Rho - rotate each lane in place */ + z8 = _mm512_rol_epi64(z8, 1); + z11 = _mm512_rol_epi64(z11, 62); + z19 = _mm512_rol_epi64(z19, 28); + z22 = _mm512_rol_epi64(z22, 27); + z13 = _mm512_rol_epi64(z13, 36); + z16 = _mm512_rol_epi64(z16, 44); + z24 = _mm512_rol_epi64(z24, 6); + z2 = _mm512_rol_epi64(z2, 55); + z5 = _mm512_rol_epi64(z5, 20); + z21 = _mm512_rol_epi64(z21, 3); + z4 = _mm512_rol_epi64(z4, 10); + z7 = _mm512_rol_epi64(z7, 43); + z10 = _mm512_rol_epi64(z10, 25); + z18 = _mm512_rol_epi64(z18, 39); + z9 = _mm512_rol_epi64(z9, 41); + z12 = _mm512_rol_epi64(z12, 45); + z15 = _mm512_rol_epi64(z15, 15); + z23 = _mm512_rol_epi64(z23, 21); + z1 = _mm512_rol_epi64(z1, 8); + z17 = _mm512_rol_epi64(z17, 18); + z20 = _mm512_rol_epi64(z20, 2); + z3 = _mm512_rol_epi64(z3, 61); + z6 = _mm512_rol_epi64(z6, 56); + z14 = _mm512_rol_epi64(z14, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z16; + z0 = _mm512_ternarylogic_epi64(z0, z16, z7, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z7, z23, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z23, z14, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z19; + z31 = z5; + z19 = _mm512_ternarylogic_epi64(z19, z5, z21, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z21, z12, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z12, z3, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z8; + z31 = z24; + z8 = _mm512_ternarylogic_epi64(z8, z24, z10, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z10, z1, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z1, z17, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z22; + z31 = z13; + z22 = _mm512_ternarylogic_epi64(z22, z13, z4, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z4, z15, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z15, z6, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z11; + z31 = z2; + z11 = _mm512_ternarylogic_epi64(z11, z2, z18, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z18, z9, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z9, z20, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 960))); + /* Round 16 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z19, z8, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z22, z11, 0x96); + z26 = z16; + z26 = _mm512_ternarylogic_epi64(z26, z5, z24, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z13, z2, 0x96); + z27 = z7; + z27 = _mm512_ternarylogic_epi64(z27, z21, z10, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z4, z18, 0x96); + z28 = z23; + z28 = _mm512_ternarylogic_epi64(z28, z12, z1, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z15, z9, 0x96); + z29 = z14; + z29 = _mm512_ternarylogic_epi64(z29, z3, z17, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z6, z20, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z19 = _mm512_xor_si512(z19, z30); + z8 = _mm512_xor_si512(z8, z30); + z22 = _mm512_xor_si512(z22, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z16 = _mm512_xor_si512(z16, z30); + z5 = _mm512_xor_si512(z5, z30); + z24 = _mm512_xor_si512(z24, z30); + z13 = _mm512_xor_si512(z13, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z7 = _mm512_xor_si512(z7, z30); + z21 = _mm512_xor_si512(z21, z30); + z10 = _mm512_xor_si512(z10, z30); + z4 = _mm512_xor_si512(z4, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z23 = _mm512_xor_si512(z23, z30); + z12 = _mm512_xor_si512(z12, z30); + z1 = _mm512_xor_si512(z1, z30); + z15 = _mm512_xor_si512(z15, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z14 = _mm512_xor_si512(z14, z30); + z3 = _mm512_xor_si512(z3, z30); + z17 = _mm512_xor_si512(z17, z30); + z6 = _mm512_xor_si512(z6, z30); + z20 = _mm512_xor_si512(z20, z30); + /* Rho - rotate each lane in place */ + z16 = _mm512_rol_epi64(z16, 1); + z7 = _mm512_rol_epi64(z7, 62); + z23 = _mm512_rol_epi64(z23, 28); + z14 = _mm512_rol_epi64(z14, 27); + z19 = _mm512_rol_epi64(z19, 36); + z5 = _mm512_rol_epi64(z5, 44); + z21 = _mm512_rol_epi64(z21, 6); + z12 = _mm512_rol_epi64(z12, 55); + z3 = _mm512_rol_epi64(z3, 20); + z8 = _mm512_rol_epi64(z8, 3); + z24 = _mm512_rol_epi64(z24, 10); + z10 = _mm512_rol_epi64(z10, 43); + z1 = _mm512_rol_epi64(z1, 25); + z17 = _mm512_rol_epi64(z17, 39); + z22 = _mm512_rol_epi64(z22, 41); + z13 = _mm512_rol_epi64(z13, 45); + z4 = _mm512_rol_epi64(z4, 15); + z15 = _mm512_rol_epi64(z15, 21); + z6 = _mm512_rol_epi64(z6, 8); + z11 = _mm512_rol_epi64(z11, 18); + z2 = _mm512_rol_epi64(z2, 2); + z18 = _mm512_rol_epi64(z18, 61); + z9 = _mm512_rol_epi64(z9, 56); + z20 = _mm512_rol_epi64(z20, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z5; + z0 = _mm512_ternarylogic_epi64(z0, z5, z10, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z10, z15, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z15, z20, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z23; + z31 = z3; + z23 = _mm512_ternarylogic_epi64(z23, z3, z8, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z8, z13, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z13, z18, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z16; + z31 = z21; + z16 = _mm512_ternarylogic_epi64(z16, z21, z1, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z1, z6, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z6, z11, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z14; + z31 = z19; + z14 = _mm512_ternarylogic_epi64(z14, z19, z24, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z24, z4, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z4, z9, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z7; + z31 = z12; + z7 = _mm512_ternarylogic_epi64(z7, z12, z17, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z17, z22, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z22, z2, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1024))); + /* Round 17 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z23, z16, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z14, z7, 0x96); + z26 = z5; + z26 = _mm512_ternarylogic_epi64(z26, z3, z21, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z19, z12, 0x96); + z27 = z10; + z27 = _mm512_ternarylogic_epi64(z27, z8, z1, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z24, z17, 0x96); + z28 = z15; + z28 = _mm512_ternarylogic_epi64(z28, z13, z6, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z4, z22, 0x96); + z29 = z20; + z29 = _mm512_ternarylogic_epi64(z29, z18, z11, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z9, z2, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z23 = _mm512_xor_si512(z23, z30); + z16 = _mm512_xor_si512(z16, z30); + z14 = _mm512_xor_si512(z14, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z5 = _mm512_xor_si512(z5, z30); + z3 = _mm512_xor_si512(z3, z30); + z21 = _mm512_xor_si512(z21, z30); + z19 = _mm512_xor_si512(z19, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z10 = _mm512_xor_si512(z10, z30); + z8 = _mm512_xor_si512(z8, z30); + z1 = _mm512_xor_si512(z1, z30); + z24 = _mm512_xor_si512(z24, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z15 = _mm512_xor_si512(z15, z30); + z13 = _mm512_xor_si512(z13, z30); + z6 = _mm512_xor_si512(z6, z30); + z4 = _mm512_xor_si512(z4, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z20 = _mm512_xor_si512(z20, z30); + z18 = _mm512_xor_si512(z18, z30); + z11 = _mm512_xor_si512(z11, z30); + z9 = _mm512_xor_si512(z9, z30); + z2 = _mm512_xor_si512(z2, z30); + /* Rho - rotate each lane in place */ + z5 = _mm512_rol_epi64(z5, 1); + z10 = _mm512_rol_epi64(z10, 62); + z15 = _mm512_rol_epi64(z15, 28); + z20 = _mm512_rol_epi64(z20, 27); + z23 = _mm512_rol_epi64(z23, 36); + z3 = _mm512_rol_epi64(z3, 44); + z8 = _mm512_rol_epi64(z8, 6); + z13 = _mm512_rol_epi64(z13, 55); + z18 = _mm512_rol_epi64(z18, 20); + z16 = _mm512_rol_epi64(z16, 3); + z21 = _mm512_rol_epi64(z21, 10); + z1 = _mm512_rol_epi64(z1, 43); + z6 = _mm512_rol_epi64(z6, 25); + z11 = _mm512_rol_epi64(z11, 39); + z14 = _mm512_rol_epi64(z14, 41); + z19 = _mm512_rol_epi64(z19, 45); + z24 = _mm512_rol_epi64(z24, 15); + z4 = _mm512_rol_epi64(z4, 21); + z9 = _mm512_rol_epi64(z9, 8); + z7 = _mm512_rol_epi64(z7, 18); + z12 = _mm512_rol_epi64(z12, 2); + z17 = _mm512_rol_epi64(z17, 61); + z22 = _mm512_rol_epi64(z22, 56); + z2 = _mm512_rol_epi64(z2, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z3; + z0 = _mm512_ternarylogic_epi64(z0, z3, z1, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z1, z4, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z4, z2, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z15; + z31 = z18; + z15 = _mm512_ternarylogic_epi64(z15, z18, z16, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z16, z19, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z19, z17, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z5; + z31 = z8; + z5 = _mm512_ternarylogic_epi64(z5, z8, z6, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z6, z9, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z9, z7, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z20; + z31 = z23; + z20 = _mm512_ternarylogic_epi64(z20, z23, z21, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z21, z24, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z24, z22, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z10; + z31 = z13; + z10 = _mm512_ternarylogic_epi64(z10, z13, z11, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z11, z14, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z14, z12, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1088))); + /* Round 18 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z15, z5, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z20, z10, 0x96); + z26 = z3; + z26 = _mm512_ternarylogic_epi64(z26, z18, z8, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z23, z13, 0x96); + z27 = z1; + z27 = _mm512_ternarylogic_epi64(z27, z16, z6, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z21, z11, 0x96); + z28 = z4; + z28 = _mm512_ternarylogic_epi64(z28, z19, z9, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z24, z14, 0x96); + z29 = z2; + z29 = _mm512_ternarylogic_epi64(z29, z17, z7, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z22, z12, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z15 = _mm512_xor_si512(z15, z30); + z5 = _mm512_xor_si512(z5, z30); + z20 = _mm512_xor_si512(z20, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z3 = _mm512_xor_si512(z3, z30); + z18 = _mm512_xor_si512(z18, z30); + z8 = _mm512_xor_si512(z8, z30); + z23 = _mm512_xor_si512(z23, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z1 = _mm512_xor_si512(z1, z30); + z16 = _mm512_xor_si512(z16, z30); + z6 = _mm512_xor_si512(z6, z30); + z21 = _mm512_xor_si512(z21, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z4 = _mm512_xor_si512(z4, z30); + z19 = _mm512_xor_si512(z19, z30); + z9 = _mm512_xor_si512(z9, z30); + z24 = _mm512_xor_si512(z24, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z2 = _mm512_xor_si512(z2, z30); + z17 = _mm512_xor_si512(z17, z30); + z7 = _mm512_xor_si512(z7, z30); + z22 = _mm512_xor_si512(z22, z30); + z12 = _mm512_xor_si512(z12, z30); + /* Rho - rotate each lane in place */ + z3 = _mm512_rol_epi64(z3, 1); + z1 = _mm512_rol_epi64(z1, 62); + z4 = _mm512_rol_epi64(z4, 28); + z2 = _mm512_rol_epi64(z2, 27); + z15 = _mm512_rol_epi64(z15, 36); + z18 = _mm512_rol_epi64(z18, 44); + z16 = _mm512_rol_epi64(z16, 6); + z19 = _mm512_rol_epi64(z19, 55); + z17 = _mm512_rol_epi64(z17, 20); + z5 = _mm512_rol_epi64(z5, 3); + z8 = _mm512_rol_epi64(z8, 10); + z6 = _mm512_rol_epi64(z6, 43); + z9 = _mm512_rol_epi64(z9, 25); + z7 = _mm512_rol_epi64(z7, 39); + z20 = _mm512_rol_epi64(z20, 41); + z23 = _mm512_rol_epi64(z23, 45); + z21 = _mm512_rol_epi64(z21, 15); + z24 = _mm512_rol_epi64(z24, 21); + z22 = _mm512_rol_epi64(z22, 8); + z10 = _mm512_rol_epi64(z10, 18); + z13 = _mm512_rol_epi64(z13, 2); + z11 = _mm512_rol_epi64(z11, 61); + z14 = _mm512_rol_epi64(z14, 56); + z12 = _mm512_rol_epi64(z12, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z18; + z0 = _mm512_ternarylogic_epi64(z0, z18, z6, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z6, z24, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z24, z12, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z4; + z31 = z17; + z4 = _mm512_ternarylogic_epi64(z4, z17, z5, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z5, z23, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z23, z11, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z3; + z31 = z16; + z3 = _mm512_ternarylogic_epi64(z3, z16, z9, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z9, z22, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z22, z10, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z2; + z31 = z15; + z2 = _mm512_ternarylogic_epi64(z2, z15, z8, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z8, z21, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z21, z14, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z1; + z31 = z19; + z1 = _mm512_ternarylogic_epi64(z1, z19, z7, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z7, z20, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z20, z13, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1152))); + /* Round 19 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z4, z3, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z2, z1, 0x96); + z26 = z18; + z26 = _mm512_ternarylogic_epi64(z26, z17, z16, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z15, z19, 0x96); + z27 = z6; + z27 = _mm512_ternarylogic_epi64(z27, z5, z9, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z8, z7, 0x96); + z28 = z24; + z28 = _mm512_ternarylogic_epi64(z28, z23, z22, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z21, z20, 0x96); + z29 = z12; + z29 = _mm512_ternarylogic_epi64(z29, z11, z10, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z14, z13, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z4 = _mm512_xor_si512(z4, z30); + z3 = _mm512_xor_si512(z3, z30); + z2 = _mm512_xor_si512(z2, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z18 = _mm512_xor_si512(z18, z30); + z17 = _mm512_xor_si512(z17, z30); + z16 = _mm512_xor_si512(z16, z30); + z15 = _mm512_xor_si512(z15, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z6 = _mm512_xor_si512(z6, z30); + z5 = _mm512_xor_si512(z5, z30); + z9 = _mm512_xor_si512(z9, z30); + z8 = _mm512_xor_si512(z8, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z24 = _mm512_xor_si512(z24, z30); + z23 = _mm512_xor_si512(z23, z30); + z22 = _mm512_xor_si512(z22, z30); + z21 = _mm512_xor_si512(z21, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z12 = _mm512_xor_si512(z12, z30); + z11 = _mm512_xor_si512(z11, z30); + z10 = _mm512_xor_si512(z10, z30); + z14 = _mm512_xor_si512(z14, z30); + z13 = _mm512_xor_si512(z13, z30); + /* Rho - rotate each lane in place */ + z18 = _mm512_rol_epi64(z18, 1); + z6 = _mm512_rol_epi64(z6, 62); + z24 = _mm512_rol_epi64(z24, 28); + z12 = _mm512_rol_epi64(z12, 27); + z4 = _mm512_rol_epi64(z4, 36); + z17 = _mm512_rol_epi64(z17, 44); + z5 = _mm512_rol_epi64(z5, 6); + z23 = _mm512_rol_epi64(z23, 55); + z11 = _mm512_rol_epi64(z11, 20); + z3 = _mm512_rol_epi64(z3, 3); + z16 = _mm512_rol_epi64(z16, 10); + z9 = _mm512_rol_epi64(z9, 43); + z22 = _mm512_rol_epi64(z22, 25); + z10 = _mm512_rol_epi64(z10, 39); + z2 = _mm512_rol_epi64(z2, 41); + z15 = _mm512_rol_epi64(z15, 45); + z8 = _mm512_rol_epi64(z8, 15); + z21 = _mm512_rol_epi64(z21, 21); + z14 = _mm512_rol_epi64(z14, 8); + z1 = _mm512_rol_epi64(z1, 18); + z19 = _mm512_rol_epi64(z19, 2); + z7 = _mm512_rol_epi64(z7, 61); + z20 = _mm512_rol_epi64(z20, 56); + z13 = _mm512_rol_epi64(z13, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z17; + z0 = _mm512_ternarylogic_epi64(z0, z17, z9, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z9, z21, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z21, z13, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z24; + z31 = z11; + z24 = _mm512_ternarylogic_epi64(z24, z11, z3, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z3, z15, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z15, z7, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z18; + z31 = z5; + z18 = _mm512_ternarylogic_epi64(z18, z5, z22, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z22, z14, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z14, z1, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z12; + z31 = z4; + z12 = _mm512_ternarylogic_epi64(z12, z4, z16, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z16, z8, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z8, z20, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z6; + z31 = z23; + z6 = _mm512_ternarylogic_epi64(z6, z23, z10, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z10, z2, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z2, z19, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1216))); + /* Round 20 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z24, z18, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z12, z6, 0x96); + z26 = z17; + z26 = _mm512_ternarylogic_epi64(z26, z11, z5, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z4, z23, 0x96); + z27 = z9; + z27 = _mm512_ternarylogic_epi64(z27, z3, z22, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z16, z10, 0x96); + z28 = z21; + z28 = _mm512_ternarylogic_epi64(z28, z15, z14, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z8, z2, 0x96); + z29 = z13; + z29 = _mm512_ternarylogic_epi64(z29, z7, z1, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z20, z19, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z24 = _mm512_xor_si512(z24, z30); + z18 = _mm512_xor_si512(z18, z30); + z12 = _mm512_xor_si512(z12, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z17 = _mm512_xor_si512(z17, z30); + z11 = _mm512_xor_si512(z11, z30); + z5 = _mm512_xor_si512(z5, z30); + z4 = _mm512_xor_si512(z4, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z9 = _mm512_xor_si512(z9, z30); + z3 = _mm512_xor_si512(z3, z30); + z22 = _mm512_xor_si512(z22, z30); + z16 = _mm512_xor_si512(z16, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z21 = _mm512_xor_si512(z21, z30); + z15 = _mm512_xor_si512(z15, z30); + z14 = _mm512_xor_si512(z14, z30); + z8 = _mm512_xor_si512(z8, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z13 = _mm512_xor_si512(z13, z30); + z7 = _mm512_xor_si512(z7, z30); + z1 = _mm512_xor_si512(z1, z30); + z20 = _mm512_xor_si512(z20, z30); + z19 = _mm512_xor_si512(z19, z30); + /* Rho - rotate each lane in place */ + z17 = _mm512_rol_epi64(z17, 1); + z9 = _mm512_rol_epi64(z9, 62); + z21 = _mm512_rol_epi64(z21, 28); + z13 = _mm512_rol_epi64(z13, 27); + z24 = _mm512_rol_epi64(z24, 36); + z11 = _mm512_rol_epi64(z11, 44); + z3 = _mm512_rol_epi64(z3, 6); + z15 = _mm512_rol_epi64(z15, 55); + z7 = _mm512_rol_epi64(z7, 20); + z18 = _mm512_rol_epi64(z18, 3); + z5 = _mm512_rol_epi64(z5, 10); + z22 = _mm512_rol_epi64(z22, 43); + z14 = _mm512_rol_epi64(z14, 25); + z1 = _mm512_rol_epi64(z1, 39); + z12 = _mm512_rol_epi64(z12, 41); + z4 = _mm512_rol_epi64(z4, 45); + z16 = _mm512_rol_epi64(z16, 15); + z8 = _mm512_rol_epi64(z8, 21); + z20 = _mm512_rol_epi64(z20, 8); + z6 = _mm512_rol_epi64(z6, 18); + z23 = _mm512_rol_epi64(z23, 2); + z10 = _mm512_rol_epi64(z10, 61); + z2 = _mm512_rol_epi64(z2, 56); + z19 = _mm512_rol_epi64(z19, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z11; + z0 = _mm512_ternarylogic_epi64(z0, z11, z22, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z22, z8, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z8, z19, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z21; + z31 = z7; + z21 = _mm512_ternarylogic_epi64(z21, z7, z18, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z18, z4, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z4, z10, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z17; + z31 = z3; + z17 = _mm512_ternarylogic_epi64(z17, z3, z14, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z14, z20, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z20, z6, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z13; + z31 = z24; + z13 = _mm512_ternarylogic_epi64(z13, z24, z5, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z5, z16, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z16, z2, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z9; + z31 = z15; + z9 = _mm512_ternarylogic_epi64(z9, z15, z1, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z1, z12, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z12, z23, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1280))); + /* Round 21 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z21, z17, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z13, z9, 0x96); + z26 = z11; + z26 = _mm512_ternarylogic_epi64(z26, z7, z3, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z24, z15, 0x96); + z27 = z22; + z27 = _mm512_ternarylogic_epi64(z27, z18, z14, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z5, z1, 0x96); + z28 = z8; + z28 = _mm512_ternarylogic_epi64(z28, z4, z20, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z16, z12, 0x96); + z29 = z19; + z29 = _mm512_ternarylogic_epi64(z29, z10, z6, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z2, z23, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z21 = _mm512_xor_si512(z21, z30); + z17 = _mm512_xor_si512(z17, z30); + z13 = _mm512_xor_si512(z13, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z11 = _mm512_xor_si512(z11, z30); + z7 = _mm512_xor_si512(z7, z30); + z3 = _mm512_xor_si512(z3, z30); + z24 = _mm512_xor_si512(z24, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z22 = _mm512_xor_si512(z22, z30); + z18 = _mm512_xor_si512(z18, z30); + z14 = _mm512_xor_si512(z14, z30); + z5 = _mm512_xor_si512(z5, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z8 = _mm512_xor_si512(z8, z30); + z4 = _mm512_xor_si512(z4, z30); + z20 = _mm512_xor_si512(z20, z30); + z16 = _mm512_xor_si512(z16, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z19 = _mm512_xor_si512(z19, z30); + z10 = _mm512_xor_si512(z10, z30); + z6 = _mm512_xor_si512(z6, z30); + z2 = _mm512_xor_si512(z2, z30); + z23 = _mm512_xor_si512(z23, z30); + /* Rho - rotate each lane in place */ + z11 = _mm512_rol_epi64(z11, 1); + z22 = _mm512_rol_epi64(z22, 62); + z8 = _mm512_rol_epi64(z8, 28); + z19 = _mm512_rol_epi64(z19, 27); + z21 = _mm512_rol_epi64(z21, 36); + z7 = _mm512_rol_epi64(z7, 44); + z18 = _mm512_rol_epi64(z18, 6); + z4 = _mm512_rol_epi64(z4, 55); + z10 = _mm512_rol_epi64(z10, 20); + z17 = _mm512_rol_epi64(z17, 3); + z3 = _mm512_rol_epi64(z3, 10); + z14 = _mm512_rol_epi64(z14, 43); + z20 = _mm512_rol_epi64(z20, 25); + z6 = _mm512_rol_epi64(z6, 39); + z13 = _mm512_rol_epi64(z13, 41); + z24 = _mm512_rol_epi64(z24, 45); + z5 = _mm512_rol_epi64(z5, 15); + z16 = _mm512_rol_epi64(z16, 21); + z2 = _mm512_rol_epi64(z2, 8); + z9 = _mm512_rol_epi64(z9, 18); + z15 = _mm512_rol_epi64(z15, 2); + z1 = _mm512_rol_epi64(z1, 61); + z12 = _mm512_rol_epi64(z12, 56); + z23 = _mm512_rol_epi64(z23, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z7; + z0 = _mm512_ternarylogic_epi64(z0, z7, z14, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z14, z16, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z16, z23, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z8; + z31 = z10; + z8 = _mm512_ternarylogic_epi64(z8, z10, z17, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z17, z24, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z24, z1, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z11; + z31 = z18; + z11 = _mm512_ternarylogic_epi64(z11, z18, z20, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z20, z2, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z2, z9, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z19; + z31 = z21; + z19 = _mm512_ternarylogic_epi64(z19, z21, z3, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z3, z5, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z5, z12, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z22; + z31 = z4; + z22 = _mm512_ternarylogic_epi64(z22, z4, z6, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z6, z13, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z13, z15, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1344))); + /* Round 22 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z8, z11, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z19, z22, 0x96); + z26 = z7; + z26 = _mm512_ternarylogic_epi64(z26, z10, z18, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z21, z4, 0x96); + z27 = z14; + z27 = _mm512_ternarylogic_epi64(z27, z17, z20, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z3, z6, 0x96); + z28 = z16; + z28 = _mm512_ternarylogic_epi64(z28, z24, z2, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z5, z13, 0x96); + z29 = z23; + z29 = _mm512_ternarylogic_epi64(z29, z1, z9, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z12, z15, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z8 = _mm512_xor_si512(z8, z30); + z11 = _mm512_xor_si512(z11, z30); + z19 = _mm512_xor_si512(z19, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z7 = _mm512_xor_si512(z7, z30); + z10 = _mm512_xor_si512(z10, z30); + z18 = _mm512_xor_si512(z18, z30); + z21 = _mm512_xor_si512(z21, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z14 = _mm512_xor_si512(z14, z30); + z17 = _mm512_xor_si512(z17, z30); + z20 = _mm512_xor_si512(z20, z30); + z3 = _mm512_xor_si512(z3, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z16 = _mm512_xor_si512(z16, z30); + z24 = _mm512_xor_si512(z24, z30); + z2 = _mm512_xor_si512(z2, z30); + z5 = _mm512_xor_si512(z5, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z23 = _mm512_xor_si512(z23, z30); + z1 = _mm512_xor_si512(z1, z30); + z9 = _mm512_xor_si512(z9, z30); + z12 = _mm512_xor_si512(z12, z30); + z15 = _mm512_xor_si512(z15, z30); + /* Rho - rotate each lane in place */ + z7 = _mm512_rol_epi64(z7, 1); + z14 = _mm512_rol_epi64(z14, 62); + z16 = _mm512_rol_epi64(z16, 28); + z23 = _mm512_rol_epi64(z23, 27); + z8 = _mm512_rol_epi64(z8, 36); + z10 = _mm512_rol_epi64(z10, 44); + z17 = _mm512_rol_epi64(z17, 6); + z24 = _mm512_rol_epi64(z24, 55); + z1 = _mm512_rol_epi64(z1, 20); + z11 = _mm512_rol_epi64(z11, 3); + z18 = _mm512_rol_epi64(z18, 10); + z20 = _mm512_rol_epi64(z20, 43); + z2 = _mm512_rol_epi64(z2, 25); + z9 = _mm512_rol_epi64(z9, 39); + z19 = _mm512_rol_epi64(z19, 41); + z21 = _mm512_rol_epi64(z21, 45); + z3 = _mm512_rol_epi64(z3, 15); + z5 = _mm512_rol_epi64(z5, 21); + z12 = _mm512_rol_epi64(z12, 8); + z22 = _mm512_rol_epi64(z22, 18); + z4 = _mm512_rol_epi64(z4, 2); + z6 = _mm512_rol_epi64(z6, 61); + z13 = _mm512_rol_epi64(z13, 56); + z15 = _mm512_rol_epi64(z15, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z10; + z0 = _mm512_ternarylogic_epi64(z0, z10, z20, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z20, z5, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z5, z15, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z16; + z31 = z1; + z16 = _mm512_ternarylogic_epi64(z16, z1, z11, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z11, z21, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z21, z6, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z7; + z31 = z17; + z7 = _mm512_ternarylogic_epi64(z7, z17, z2, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z2, z12, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z12, z22, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z23; + z31 = z8; + z23 = _mm512_ternarylogic_epi64(z23, z8, z18, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z18, z3, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z3, z13, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z14; + z31 = z24; + z14 = _mm512_ternarylogic_epi64(z14, z24, z9, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z9, z19, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z19, z4, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1408))); + /* Round 23 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z16, z7, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z23, z14, 0x96); + z26 = z10; + z26 = _mm512_ternarylogic_epi64(z26, z1, z17, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z8, z24, 0x96); + z27 = z20; + z27 = _mm512_ternarylogic_epi64(z27, z11, z2, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z18, z9, 0x96); + z28 = z5; + z28 = _mm512_ternarylogic_epi64(z28, z21, z12, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z3, z19, 0x96); + z29 = z15; + z29 = _mm512_ternarylogic_epi64(z29, z6, z22, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z13, z4, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z16 = _mm512_xor_si512(z16, z30); + z7 = _mm512_xor_si512(z7, z30); + z23 = _mm512_xor_si512(z23, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z10 = _mm512_xor_si512(z10, z30); + z1 = _mm512_xor_si512(z1, z30); + z17 = _mm512_xor_si512(z17, z30); + z8 = _mm512_xor_si512(z8, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z20 = _mm512_xor_si512(z20, z30); + z11 = _mm512_xor_si512(z11, z30); + z2 = _mm512_xor_si512(z2, z30); + z18 = _mm512_xor_si512(z18, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z5 = _mm512_xor_si512(z5, z30); + z21 = _mm512_xor_si512(z21, z30); + z12 = _mm512_xor_si512(z12, z30); + z3 = _mm512_xor_si512(z3, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z15 = _mm512_xor_si512(z15, z30); + z6 = _mm512_xor_si512(z6, z30); + z22 = _mm512_xor_si512(z22, z30); + z13 = _mm512_xor_si512(z13, z30); + z4 = _mm512_xor_si512(z4, z30); + /* Rho - rotate each lane in place */ + z10 = _mm512_rol_epi64(z10, 1); + z20 = _mm512_rol_epi64(z20, 62); + z5 = _mm512_rol_epi64(z5, 28); + z15 = _mm512_rol_epi64(z15, 27); + z16 = _mm512_rol_epi64(z16, 36); + z1 = _mm512_rol_epi64(z1, 44); + z11 = _mm512_rol_epi64(z11, 6); + z21 = _mm512_rol_epi64(z21, 55); + z6 = _mm512_rol_epi64(z6, 20); + z7 = _mm512_rol_epi64(z7, 3); + z17 = _mm512_rol_epi64(z17, 10); + z2 = _mm512_rol_epi64(z2, 43); + z12 = _mm512_rol_epi64(z12, 25); + z22 = _mm512_rol_epi64(z22, 39); + z23 = _mm512_rol_epi64(z23, 41); + z8 = _mm512_rol_epi64(z8, 45); + z18 = _mm512_rol_epi64(z18, 15); + z3 = _mm512_rol_epi64(z3, 21); + z13 = _mm512_rol_epi64(z13, 8); + z14 = _mm512_rol_epi64(z14, 18); + z24 = _mm512_rol_epi64(z24, 2); + z9 = _mm512_rol_epi64(z9, 61); + z19 = _mm512_rol_epi64(z19, 56); + z4 = _mm512_rol_epi64(z4, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z1; + z0 = _mm512_ternarylogic_epi64(z0, z1, z2, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z2, z3, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z3, z4, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z5; + z31 = z6; + z5 = _mm512_ternarylogic_epi64(z5, z6, z7, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z7, z8, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z8, z9, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z10; + z31 = z11; + z10 = _mm512_ternarylogic_epi64(z10, z11, z12, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z12, z13, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z13, z14, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z15; + z31 = z16; + z15 = _mm512_ternarylogic_epi64(z15, z16, z17, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z17, z18, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z18, z19, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z20; + z31 = z21; + z20 = _mm512_ternarylogic_epi64(z20, z21, z22, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z22, z23, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z23, z24, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1472))); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 1024), z16); + _mm512_storeu_si512((void*)WC_PW(rdi, 1088), z17); + _mm512_storeu_si512((void*)WC_PW(rdi, 1152), z18); + _mm512_storeu_si512((void*)WC_PW(rdi, 1216), z19); + _mm512_storeu_si512((void*)WC_PW(rdi, 1280), z20); + _mm512_storeu_si512((void*)WC_PW(rdi, 1344), z21); + _mm512_storeu_si512((void*)WC_PW(rdi, 1408), z22); + _mm512_storeu_si512((void*)WC_PW(rdi, 1472), z23); + _mm512_storeu_si512((void*)WC_PW(rdi, 1536), z24); +} + +#endif /* defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) */ +#ifdef WOLFSSL_HAVE_MLKEM +XALIGNED(32) static const word64 L_sha3_256_blocksx8_seed_avx512_end_mark[] + WC_X64I_UNUSED = { + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void sha3_256_blocksx8_seed_avx512(word64* s, byte* seed) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23, z24, z25, z26, z27, + z28, z29, z30, z31; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)seed; + + rdx = (word64)(L_sha3_x8_avx512_r); + z0 = _mm512_set1_epi64((long long)WC_L64(rsi, 0)); + z1 = _mm512_set1_epi64((long long)WC_L64(rsi, 8)); + z2 = _mm512_set1_epi64((long long)WC_L64(rsi, 16)); + z3 = _mm512_set1_epi64((long long)WC_L64(rsi, 24)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_setzero_si512(); + z6 = z5; + z7 = z5; + z8 = z5; + z9 = z5; + z10 = z5; + z11 = z5; + z12 = z5; + z13 = z5; + z14 = z5; + z15 = z5; + z16 = _mm512_loadu_si512((const void*)WC_PR( + L_sha3_256_blocksx8_seed_avx512_end_mark, 0)); + z17 = z5; + z18 = z5; + z19 = z5; + z20 = z5; + z21 = z5; + z22 = z5; + z23 = z5; + z24 = z5; + /* Round 0 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z5, z10, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z15, z20, 0x96); + z26 = z1; + z26 = _mm512_ternarylogic_epi64(z26, z6, z11, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z16, z21, 0x96); + z27 = z2; + z27 = _mm512_ternarylogic_epi64(z27, z7, z12, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z17, z22, 0x96); + z28 = z3; + z28 = _mm512_ternarylogic_epi64(z28, z8, z13, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z18, z23, 0x96); + z29 = z4; + z29 = _mm512_ternarylogic_epi64(z29, z9, z14, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z19, z24, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z5 = _mm512_xor_si512(z5, z30); + z10 = _mm512_xor_si512(z10, z30); + z15 = _mm512_xor_si512(z15, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z1 = _mm512_xor_si512(z1, z30); + z6 = _mm512_xor_si512(z6, z30); + z11 = _mm512_xor_si512(z11, z30); + z16 = _mm512_xor_si512(z16, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z2 = _mm512_xor_si512(z2, z30); + z7 = _mm512_xor_si512(z7, z30); + z12 = _mm512_xor_si512(z12, z30); + z17 = _mm512_xor_si512(z17, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z3 = _mm512_xor_si512(z3, z30); + z8 = _mm512_xor_si512(z8, z30); + z13 = _mm512_xor_si512(z13, z30); + z18 = _mm512_xor_si512(z18, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z4 = _mm512_xor_si512(z4, z30); + z9 = _mm512_xor_si512(z9, z30); + z14 = _mm512_xor_si512(z14, z30); + z19 = _mm512_xor_si512(z19, z30); + z24 = _mm512_xor_si512(z24, z30); + /* Rho - rotate each lane in place */ + z1 = _mm512_rol_epi64(z1, 1); + z2 = _mm512_rol_epi64(z2, 62); + z3 = _mm512_rol_epi64(z3, 28); + z4 = _mm512_rol_epi64(z4, 27); + z5 = _mm512_rol_epi64(z5, 36); + z6 = _mm512_rol_epi64(z6, 44); + z7 = _mm512_rol_epi64(z7, 6); + z8 = _mm512_rol_epi64(z8, 55); + z9 = _mm512_rol_epi64(z9, 20); + z10 = _mm512_rol_epi64(z10, 3); + z11 = _mm512_rol_epi64(z11, 10); + z12 = _mm512_rol_epi64(z12, 43); + z13 = _mm512_rol_epi64(z13, 25); + z14 = _mm512_rol_epi64(z14, 39); + z15 = _mm512_rol_epi64(z15, 41); + z16 = _mm512_rol_epi64(z16, 45); + z17 = _mm512_rol_epi64(z17, 15); + z18 = _mm512_rol_epi64(z18, 21); + z19 = _mm512_rol_epi64(z19, 8); + z20 = _mm512_rol_epi64(z20, 18); + z21 = _mm512_rol_epi64(z21, 2); + z22 = _mm512_rol_epi64(z22, 61); + z23 = _mm512_rol_epi64(z23, 56); + z24 = _mm512_rol_epi64(z24, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z6; + z0 = _mm512_ternarylogic_epi64(z0, z6, z12, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z12, z18, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z18, z24, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z3; + z31 = z9; + z3 = _mm512_ternarylogic_epi64(z3, z9, z10, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z10, z16, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z16, z22, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z1; + z31 = z7; + z1 = _mm512_ternarylogic_epi64(z1, z7, z13, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z13, z19, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z19, z20, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z4; + z31 = z5; + z4 = _mm512_ternarylogic_epi64(z4, z5, z11, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z11, z17, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z17, z23, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z2; + z31 = z8; + z2 = _mm512_ternarylogic_epi64(z2, z8, z14, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z14, z15, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z15, z21, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 0))); + /* Round 1 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z3, z1, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z4, z2, 0x96); + z26 = z6; + z26 = _mm512_ternarylogic_epi64(z26, z9, z7, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z5, z8, 0x96); + z27 = z12; + z27 = _mm512_ternarylogic_epi64(z27, z10, z13, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z11, z14, 0x96); + z28 = z18; + z28 = _mm512_ternarylogic_epi64(z28, z16, z19, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z17, z15, 0x96); + z29 = z24; + z29 = _mm512_ternarylogic_epi64(z29, z22, z20, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z23, z21, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z3 = _mm512_xor_si512(z3, z30); + z1 = _mm512_xor_si512(z1, z30); + z4 = _mm512_xor_si512(z4, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z6 = _mm512_xor_si512(z6, z30); + z9 = _mm512_xor_si512(z9, z30); + z7 = _mm512_xor_si512(z7, z30); + z5 = _mm512_xor_si512(z5, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z12 = _mm512_xor_si512(z12, z30); + z10 = _mm512_xor_si512(z10, z30); + z13 = _mm512_xor_si512(z13, z30); + z11 = _mm512_xor_si512(z11, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z18 = _mm512_xor_si512(z18, z30); + z16 = _mm512_xor_si512(z16, z30); + z19 = _mm512_xor_si512(z19, z30); + z17 = _mm512_xor_si512(z17, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z24 = _mm512_xor_si512(z24, z30); + z22 = _mm512_xor_si512(z22, z30); + z20 = _mm512_xor_si512(z20, z30); + z23 = _mm512_xor_si512(z23, z30); + z21 = _mm512_xor_si512(z21, z30); + /* Rho - rotate each lane in place */ + z6 = _mm512_rol_epi64(z6, 1); + z12 = _mm512_rol_epi64(z12, 62); + z18 = _mm512_rol_epi64(z18, 28); + z24 = _mm512_rol_epi64(z24, 27); + z3 = _mm512_rol_epi64(z3, 36); + z9 = _mm512_rol_epi64(z9, 44); + z10 = _mm512_rol_epi64(z10, 6); + z16 = _mm512_rol_epi64(z16, 55); + z22 = _mm512_rol_epi64(z22, 20); + z1 = _mm512_rol_epi64(z1, 3); + z7 = _mm512_rol_epi64(z7, 10); + z13 = _mm512_rol_epi64(z13, 43); + z19 = _mm512_rol_epi64(z19, 25); + z20 = _mm512_rol_epi64(z20, 39); + z4 = _mm512_rol_epi64(z4, 41); + z5 = _mm512_rol_epi64(z5, 45); + z11 = _mm512_rol_epi64(z11, 15); + z17 = _mm512_rol_epi64(z17, 21); + z23 = _mm512_rol_epi64(z23, 8); + z2 = _mm512_rol_epi64(z2, 18); + z8 = _mm512_rol_epi64(z8, 2); + z14 = _mm512_rol_epi64(z14, 61); + z15 = _mm512_rol_epi64(z15, 56); + z21 = _mm512_rol_epi64(z21, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z9; + z0 = _mm512_ternarylogic_epi64(z0, z9, z13, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z13, z17, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z17, z21, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z18; + z31 = z22; + z18 = _mm512_ternarylogic_epi64(z18, z22, z1, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z1, z5, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z5, z14, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z6; + z31 = z10; + z6 = _mm512_ternarylogic_epi64(z6, z10, z19, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z19, z23, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z23, z2, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z24; + z31 = z3; + z24 = _mm512_ternarylogic_epi64(z24, z3, z7, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z7, z11, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z11, z15, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z12; + z31 = z16; + z12 = _mm512_ternarylogic_epi64(z12, z16, z20, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z20, z4, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z4, z8, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 64))); + /* Round 2 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z18, z6, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z24, z12, 0x96); + z26 = z9; + z26 = _mm512_ternarylogic_epi64(z26, z22, z10, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z3, z16, 0x96); + z27 = z13; + z27 = _mm512_ternarylogic_epi64(z27, z1, z19, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z7, z20, 0x96); + z28 = z17; + z28 = _mm512_ternarylogic_epi64(z28, z5, z23, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z11, z4, 0x96); + z29 = z21; + z29 = _mm512_ternarylogic_epi64(z29, z14, z2, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z15, z8, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z18 = _mm512_xor_si512(z18, z30); + z6 = _mm512_xor_si512(z6, z30); + z24 = _mm512_xor_si512(z24, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z9 = _mm512_xor_si512(z9, z30); + z22 = _mm512_xor_si512(z22, z30); + z10 = _mm512_xor_si512(z10, z30); + z3 = _mm512_xor_si512(z3, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z13 = _mm512_xor_si512(z13, z30); + z1 = _mm512_xor_si512(z1, z30); + z19 = _mm512_xor_si512(z19, z30); + z7 = _mm512_xor_si512(z7, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z17 = _mm512_xor_si512(z17, z30); + z5 = _mm512_xor_si512(z5, z30); + z23 = _mm512_xor_si512(z23, z30); + z11 = _mm512_xor_si512(z11, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z21 = _mm512_xor_si512(z21, z30); + z14 = _mm512_xor_si512(z14, z30); + z2 = _mm512_xor_si512(z2, z30); + z15 = _mm512_xor_si512(z15, z30); + z8 = _mm512_xor_si512(z8, z30); + /* Rho - rotate each lane in place */ + z9 = _mm512_rol_epi64(z9, 1); + z13 = _mm512_rol_epi64(z13, 62); + z17 = _mm512_rol_epi64(z17, 28); + z21 = _mm512_rol_epi64(z21, 27); + z18 = _mm512_rol_epi64(z18, 36); + z22 = _mm512_rol_epi64(z22, 44); + z1 = _mm512_rol_epi64(z1, 6); + z5 = _mm512_rol_epi64(z5, 55); + z14 = _mm512_rol_epi64(z14, 20); + z6 = _mm512_rol_epi64(z6, 3); + z10 = _mm512_rol_epi64(z10, 10); + z19 = _mm512_rol_epi64(z19, 43); + z23 = _mm512_rol_epi64(z23, 25); + z2 = _mm512_rol_epi64(z2, 39); + z24 = _mm512_rol_epi64(z24, 41); + z3 = _mm512_rol_epi64(z3, 45); + z7 = _mm512_rol_epi64(z7, 15); + z11 = _mm512_rol_epi64(z11, 21); + z15 = _mm512_rol_epi64(z15, 8); + z12 = _mm512_rol_epi64(z12, 18); + z16 = _mm512_rol_epi64(z16, 2); + z20 = _mm512_rol_epi64(z20, 61); + z4 = _mm512_rol_epi64(z4, 56); + z8 = _mm512_rol_epi64(z8, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z22; + z0 = _mm512_ternarylogic_epi64(z0, z22, z19, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z19, z11, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z11, z8, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z17; + z31 = z14; + z17 = _mm512_ternarylogic_epi64(z17, z14, z6, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z6, z3, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z3, z20, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z9; + z31 = z1; + z9 = _mm512_ternarylogic_epi64(z9, z1, z23, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z23, z15, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z15, z12, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z21; + z31 = z18; + z21 = _mm512_ternarylogic_epi64(z21, z18, z10, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z10, z7, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z7, z4, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z13; + z31 = z5; + z13 = _mm512_ternarylogic_epi64(z13, z5, z2, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z2, z24, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z24, z16, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 128))); + /* Round 3 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z17, z9, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z21, z13, 0x96); + z26 = z22; + z26 = _mm512_ternarylogic_epi64(z26, z14, z1, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z18, z5, 0x96); + z27 = z19; + z27 = _mm512_ternarylogic_epi64(z27, z6, z23, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z10, z2, 0x96); + z28 = z11; + z28 = _mm512_ternarylogic_epi64(z28, z3, z15, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z7, z24, 0x96); + z29 = z8; + z29 = _mm512_ternarylogic_epi64(z29, z20, z12, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z4, z16, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z17 = _mm512_xor_si512(z17, z30); + z9 = _mm512_xor_si512(z9, z30); + z21 = _mm512_xor_si512(z21, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z22 = _mm512_xor_si512(z22, z30); + z14 = _mm512_xor_si512(z14, z30); + z1 = _mm512_xor_si512(z1, z30); + z18 = _mm512_xor_si512(z18, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z19 = _mm512_xor_si512(z19, z30); + z6 = _mm512_xor_si512(z6, z30); + z23 = _mm512_xor_si512(z23, z30); + z10 = _mm512_xor_si512(z10, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z11 = _mm512_xor_si512(z11, z30); + z3 = _mm512_xor_si512(z3, z30); + z15 = _mm512_xor_si512(z15, z30); + z7 = _mm512_xor_si512(z7, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z8 = _mm512_xor_si512(z8, z30); + z20 = _mm512_xor_si512(z20, z30); + z12 = _mm512_xor_si512(z12, z30); + z4 = _mm512_xor_si512(z4, z30); + z16 = _mm512_xor_si512(z16, z30); + /* Rho - rotate each lane in place */ + z22 = _mm512_rol_epi64(z22, 1); + z19 = _mm512_rol_epi64(z19, 62); + z11 = _mm512_rol_epi64(z11, 28); + z8 = _mm512_rol_epi64(z8, 27); + z17 = _mm512_rol_epi64(z17, 36); + z14 = _mm512_rol_epi64(z14, 44); + z6 = _mm512_rol_epi64(z6, 6); + z3 = _mm512_rol_epi64(z3, 55); + z20 = _mm512_rol_epi64(z20, 20); + z9 = _mm512_rol_epi64(z9, 3); + z1 = _mm512_rol_epi64(z1, 10); + z23 = _mm512_rol_epi64(z23, 43); + z15 = _mm512_rol_epi64(z15, 25); + z12 = _mm512_rol_epi64(z12, 39); + z21 = _mm512_rol_epi64(z21, 41); + z18 = _mm512_rol_epi64(z18, 45); + z10 = _mm512_rol_epi64(z10, 15); + z7 = _mm512_rol_epi64(z7, 21); + z4 = _mm512_rol_epi64(z4, 8); + z13 = _mm512_rol_epi64(z13, 18); + z5 = _mm512_rol_epi64(z5, 2); + z2 = _mm512_rol_epi64(z2, 61); + z24 = _mm512_rol_epi64(z24, 56); + z16 = _mm512_rol_epi64(z16, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z14; + z0 = _mm512_ternarylogic_epi64(z0, z14, z23, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z23, z7, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z7, z16, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z11; + z31 = z20; + z11 = _mm512_ternarylogic_epi64(z11, z20, z9, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z9, z18, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z18, z2, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z22; + z31 = z6; + z22 = _mm512_ternarylogic_epi64(z22, z6, z15, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z15, z4, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z4, z13, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z8; + z31 = z17; + z8 = _mm512_ternarylogic_epi64(z8, z17, z1, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z1, z10, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z10, z24, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z19; + z31 = z3; + z19 = _mm512_ternarylogic_epi64(z19, z3, z12, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z12, z21, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z21, z5, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 192))); + /* Round 4 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z11, z22, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z8, z19, 0x96); + z26 = z14; + z26 = _mm512_ternarylogic_epi64(z26, z20, z6, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z17, z3, 0x96); + z27 = z23; + z27 = _mm512_ternarylogic_epi64(z27, z9, z15, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z1, z12, 0x96); + z28 = z7; + z28 = _mm512_ternarylogic_epi64(z28, z18, z4, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z10, z21, 0x96); + z29 = z16; + z29 = _mm512_ternarylogic_epi64(z29, z2, z13, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z24, z5, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z11 = _mm512_xor_si512(z11, z30); + z22 = _mm512_xor_si512(z22, z30); + z8 = _mm512_xor_si512(z8, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z14 = _mm512_xor_si512(z14, z30); + z20 = _mm512_xor_si512(z20, z30); + z6 = _mm512_xor_si512(z6, z30); + z17 = _mm512_xor_si512(z17, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z23 = _mm512_xor_si512(z23, z30); + z9 = _mm512_xor_si512(z9, z30); + z15 = _mm512_xor_si512(z15, z30); + z1 = _mm512_xor_si512(z1, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z7 = _mm512_xor_si512(z7, z30); + z18 = _mm512_xor_si512(z18, z30); + z4 = _mm512_xor_si512(z4, z30); + z10 = _mm512_xor_si512(z10, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z16 = _mm512_xor_si512(z16, z30); + z2 = _mm512_xor_si512(z2, z30); + z13 = _mm512_xor_si512(z13, z30); + z24 = _mm512_xor_si512(z24, z30); + z5 = _mm512_xor_si512(z5, z30); + /* Rho - rotate each lane in place */ + z14 = _mm512_rol_epi64(z14, 1); + z23 = _mm512_rol_epi64(z23, 62); + z7 = _mm512_rol_epi64(z7, 28); + z16 = _mm512_rol_epi64(z16, 27); + z11 = _mm512_rol_epi64(z11, 36); + z20 = _mm512_rol_epi64(z20, 44); + z9 = _mm512_rol_epi64(z9, 6); + z18 = _mm512_rol_epi64(z18, 55); + z2 = _mm512_rol_epi64(z2, 20); + z22 = _mm512_rol_epi64(z22, 3); + z6 = _mm512_rol_epi64(z6, 10); + z15 = _mm512_rol_epi64(z15, 43); + z4 = _mm512_rol_epi64(z4, 25); + z13 = _mm512_rol_epi64(z13, 39); + z8 = _mm512_rol_epi64(z8, 41); + z17 = _mm512_rol_epi64(z17, 45); + z1 = _mm512_rol_epi64(z1, 15); + z10 = _mm512_rol_epi64(z10, 21); + z24 = _mm512_rol_epi64(z24, 8); + z19 = _mm512_rol_epi64(z19, 18); + z3 = _mm512_rol_epi64(z3, 2); + z12 = _mm512_rol_epi64(z12, 61); + z21 = _mm512_rol_epi64(z21, 56); + z5 = _mm512_rol_epi64(z5, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z20; + z0 = _mm512_ternarylogic_epi64(z0, z20, z15, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z15, z10, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z10, z5, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z7; + z31 = z2; + z7 = _mm512_ternarylogic_epi64(z7, z2, z22, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z22, z17, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z17, z12, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z14; + z31 = z9; + z14 = _mm512_ternarylogic_epi64(z14, z9, z4, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z4, z24, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z24, z19, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z16; + z31 = z11; + z16 = _mm512_ternarylogic_epi64(z16, z11, z6, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z6, z1, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z1, z21, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z23; + z31 = z18; + z23 = _mm512_ternarylogic_epi64(z23, z18, z13, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z13, z8, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z8, z3, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 256))); + /* Round 5 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z7, z14, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z16, z23, 0x96); + z26 = z20; + z26 = _mm512_ternarylogic_epi64(z26, z2, z9, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z11, z18, 0x96); + z27 = z15; + z27 = _mm512_ternarylogic_epi64(z27, z22, z4, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z6, z13, 0x96); + z28 = z10; + z28 = _mm512_ternarylogic_epi64(z28, z17, z24, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z1, z8, 0x96); + z29 = z5; + z29 = _mm512_ternarylogic_epi64(z29, z12, z19, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z21, z3, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z7 = _mm512_xor_si512(z7, z30); + z14 = _mm512_xor_si512(z14, z30); + z16 = _mm512_xor_si512(z16, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z20 = _mm512_xor_si512(z20, z30); + z2 = _mm512_xor_si512(z2, z30); + z9 = _mm512_xor_si512(z9, z30); + z11 = _mm512_xor_si512(z11, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z15 = _mm512_xor_si512(z15, z30); + z22 = _mm512_xor_si512(z22, z30); + z4 = _mm512_xor_si512(z4, z30); + z6 = _mm512_xor_si512(z6, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z10 = _mm512_xor_si512(z10, z30); + z17 = _mm512_xor_si512(z17, z30); + z24 = _mm512_xor_si512(z24, z30); + z1 = _mm512_xor_si512(z1, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z5 = _mm512_xor_si512(z5, z30); + z12 = _mm512_xor_si512(z12, z30); + z19 = _mm512_xor_si512(z19, z30); + z21 = _mm512_xor_si512(z21, z30); + z3 = _mm512_xor_si512(z3, z30); + /* Rho - rotate each lane in place */ + z20 = _mm512_rol_epi64(z20, 1); + z15 = _mm512_rol_epi64(z15, 62); + z10 = _mm512_rol_epi64(z10, 28); + z5 = _mm512_rol_epi64(z5, 27); + z7 = _mm512_rol_epi64(z7, 36); + z2 = _mm512_rol_epi64(z2, 44); + z22 = _mm512_rol_epi64(z22, 6); + z17 = _mm512_rol_epi64(z17, 55); + z12 = _mm512_rol_epi64(z12, 20); + z14 = _mm512_rol_epi64(z14, 3); + z9 = _mm512_rol_epi64(z9, 10); + z4 = _mm512_rol_epi64(z4, 43); + z24 = _mm512_rol_epi64(z24, 25); + z19 = _mm512_rol_epi64(z19, 39); + z16 = _mm512_rol_epi64(z16, 41); + z11 = _mm512_rol_epi64(z11, 45); + z6 = _mm512_rol_epi64(z6, 15); + z1 = _mm512_rol_epi64(z1, 21); + z21 = _mm512_rol_epi64(z21, 8); + z23 = _mm512_rol_epi64(z23, 18); + z18 = _mm512_rol_epi64(z18, 2); + z13 = _mm512_rol_epi64(z13, 61); + z8 = _mm512_rol_epi64(z8, 56); + z3 = _mm512_rol_epi64(z3, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z2; + z0 = _mm512_ternarylogic_epi64(z0, z2, z4, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z4, z1, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z1, z3, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z10; + z31 = z12; + z10 = _mm512_ternarylogic_epi64(z10, z12, z14, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z14, z11, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z11, z13, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z20; + z31 = z22; + z20 = _mm512_ternarylogic_epi64(z20, z22, z24, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z24, z21, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z21, z23, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z5; + z31 = z7; + z5 = _mm512_ternarylogic_epi64(z5, z7, z9, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z9, z6, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z6, z8, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z15; + z31 = z17; + z15 = _mm512_ternarylogic_epi64(z15, z17, z19, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z19, z16, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z16, z18, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 320))); + /* Round 6 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z10, z20, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z5, z15, 0x96); + z26 = z2; + z26 = _mm512_ternarylogic_epi64(z26, z12, z22, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z7, z17, 0x96); + z27 = z4; + z27 = _mm512_ternarylogic_epi64(z27, z14, z24, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z9, z19, 0x96); + z28 = z1; + z28 = _mm512_ternarylogic_epi64(z28, z11, z21, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z6, z16, 0x96); + z29 = z3; + z29 = _mm512_ternarylogic_epi64(z29, z13, z23, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z8, z18, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z10 = _mm512_xor_si512(z10, z30); + z20 = _mm512_xor_si512(z20, z30); + z5 = _mm512_xor_si512(z5, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z2 = _mm512_xor_si512(z2, z30); + z12 = _mm512_xor_si512(z12, z30); + z22 = _mm512_xor_si512(z22, z30); + z7 = _mm512_xor_si512(z7, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z4 = _mm512_xor_si512(z4, z30); + z14 = _mm512_xor_si512(z14, z30); + z24 = _mm512_xor_si512(z24, z30); + z9 = _mm512_xor_si512(z9, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z1 = _mm512_xor_si512(z1, z30); + z11 = _mm512_xor_si512(z11, z30); + z21 = _mm512_xor_si512(z21, z30); + z6 = _mm512_xor_si512(z6, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z3 = _mm512_xor_si512(z3, z30); + z13 = _mm512_xor_si512(z13, z30); + z23 = _mm512_xor_si512(z23, z30); + z8 = _mm512_xor_si512(z8, z30); + z18 = _mm512_xor_si512(z18, z30); + /* Rho - rotate each lane in place */ + z2 = _mm512_rol_epi64(z2, 1); + z4 = _mm512_rol_epi64(z4, 62); + z1 = _mm512_rol_epi64(z1, 28); + z3 = _mm512_rol_epi64(z3, 27); + z10 = _mm512_rol_epi64(z10, 36); + z12 = _mm512_rol_epi64(z12, 44); + z14 = _mm512_rol_epi64(z14, 6); + z11 = _mm512_rol_epi64(z11, 55); + z13 = _mm512_rol_epi64(z13, 20); + z20 = _mm512_rol_epi64(z20, 3); + z22 = _mm512_rol_epi64(z22, 10); + z24 = _mm512_rol_epi64(z24, 43); + z21 = _mm512_rol_epi64(z21, 25); + z23 = _mm512_rol_epi64(z23, 39); + z5 = _mm512_rol_epi64(z5, 41); + z7 = _mm512_rol_epi64(z7, 45); + z9 = _mm512_rol_epi64(z9, 15); + z6 = _mm512_rol_epi64(z6, 21); + z8 = _mm512_rol_epi64(z8, 8); + z15 = _mm512_rol_epi64(z15, 18); + z17 = _mm512_rol_epi64(z17, 2); + z19 = _mm512_rol_epi64(z19, 61); + z16 = _mm512_rol_epi64(z16, 56); + z18 = _mm512_rol_epi64(z18, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z12; + z0 = _mm512_ternarylogic_epi64(z0, z12, z24, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z24, z6, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z6, z18, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z1; + z31 = z13; + z1 = _mm512_ternarylogic_epi64(z1, z13, z20, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z20, z7, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z7, z19, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z2; + z31 = z14; + z2 = _mm512_ternarylogic_epi64(z2, z14, z21, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z21, z8, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z8, z15, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z3; + z31 = z10; + z3 = _mm512_ternarylogic_epi64(z3, z10, z22, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z22, z9, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z9, z16, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z4; + z31 = z11; + z4 = _mm512_ternarylogic_epi64(z4, z11, z23, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z23, z5, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z5, z17, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 384))); + /* Round 7 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z1, z2, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z3, z4, 0x96); + z26 = z12; + z26 = _mm512_ternarylogic_epi64(z26, z13, z14, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z10, z11, 0x96); + z27 = z24; + z27 = _mm512_ternarylogic_epi64(z27, z20, z21, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z22, z23, 0x96); + z28 = z6; + z28 = _mm512_ternarylogic_epi64(z28, z7, z8, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z9, z5, 0x96); + z29 = z18; + z29 = _mm512_ternarylogic_epi64(z29, z19, z15, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z16, z17, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z1 = _mm512_xor_si512(z1, z30); + z2 = _mm512_xor_si512(z2, z30); + z3 = _mm512_xor_si512(z3, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z12 = _mm512_xor_si512(z12, z30); + z13 = _mm512_xor_si512(z13, z30); + z14 = _mm512_xor_si512(z14, z30); + z10 = _mm512_xor_si512(z10, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z24 = _mm512_xor_si512(z24, z30); + z20 = _mm512_xor_si512(z20, z30); + z21 = _mm512_xor_si512(z21, z30); + z22 = _mm512_xor_si512(z22, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z6 = _mm512_xor_si512(z6, z30); + z7 = _mm512_xor_si512(z7, z30); + z8 = _mm512_xor_si512(z8, z30); + z9 = _mm512_xor_si512(z9, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z18 = _mm512_xor_si512(z18, z30); + z19 = _mm512_xor_si512(z19, z30); + z15 = _mm512_xor_si512(z15, z30); + z16 = _mm512_xor_si512(z16, z30); + z17 = _mm512_xor_si512(z17, z30); + /* Rho - rotate each lane in place */ + z12 = _mm512_rol_epi64(z12, 1); + z24 = _mm512_rol_epi64(z24, 62); + z6 = _mm512_rol_epi64(z6, 28); + z18 = _mm512_rol_epi64(z18, 27); + z1 = _mm512_rol_epi64(z1, 36); + z13 = _mm512_rol_epi64(z13, 44); + z20 = _mm512_rol_epi64(z20, 6); + z7 = _mm512_rol_epi64(z7, 55); + z19 = _mm512_rol_epi64(z19, 20); + z2 = _mm512_rol_epi64(z2, 3); + z14 = _mm512_rol_epi64(z14, 10); + z21 = _mm512_rol_epi64(z21, 43); + z8 = _mm512_rol_epi64(z8, 25); + z15 = _mm512_rol_epi64(z15, 39); + z3 = _mm512_rol_epi64(z3, 41); + z10 = _mm512_rol_epi64(z10, 45); + z22 = _mm512_rol_epi64(z22, 15); + z9 = _mm512_rol_epi64(z9, 21); + z16 = _mm512_rol_epi64(z16, 8); + z4 = _mm512_rol_epi64(z4, 18); + z11 = _mm512_rol_epi64(z11, 2); + z23 = _mm512_rol_epi64(z23, 61); + z5 = _mm512_rol_epi64(z5, 56); + z17 = _mm512_rol_epi64(z17, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z13; + z0 = _mm512_ternarylogic_epi64(z0, z13, z21, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z21, z9, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z9, z17, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z6; + z31 = z19; + z6 = _mm512_ternarylogic_epi64(z6, z19, z2, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z2, z10, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z10, z23, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z12; + z31 = z20; + z12 = _mm512_ternarylogic_epi64(z12, z20, z8, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z8, z16, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z16, z4, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z18; + z31 = z1; + z18 = _mm512_ternarylogic_epi64(z18, z1, z14, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z14, z22, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z22, z5, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z24; + z31 = z7; + z24 = _mm512_ternarylogic_epi64(z24, z7, z15, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z15, z3, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z3, z11, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 448))); + /* Round 8 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z6, z12, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z18, z24, 0x96); + z26 = z13; + z26 = _mm512_ternarylogic_epi64(z26, z19, z20, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z1, z7, 0x96); + z27 = z21; + z27 = _mm512_ternarylogic_epi64(z27, z2, z8, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z14, z15, 0x96); + z28 = z9; + z28 = _mm512_ternarylogic_epi64(z28, z10, z16, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z22, z3, 0x96); + z29 = z17; + z29 = _mm512_ternarylogic_epi64(z29, z23, z4, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z5, z11, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z6 = _mm512_xor_si512(z6, z30); + z12 = _mm512_xor_si512(z12, z30); + z18 = _mm512_xor_si512(z18, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z13 = _mm512_xor_si512(z13, z30); + z19 = _mm512_xor_si512(z19, z30); + z20 = _mm512_xor_si512(z20, z30); + z1 = _mm512_xor_si512(z1, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z21 = _mm512_xor_si512(z21, z30); + z2 = _mm512_xor_si512(z2, z30); + z8 = _mm512_xor_si512(z8, z30); + z14 = _mm512_xor_si512(z14, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z9 = _mm512_xor_si512(z9, z30); + z10 = _mm512_xor_si512(z10, z30); + z16 = _mm512_xor_si512(z16, z30); + z22 = _mm512_xor_si512(z22, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z17 = _mm512_xor_si512(z17, z30); + z23 = _mm512_xor_si512(z23, z30); + z4 = _mm512_xor_si512(z4, z30); + z5 = _mm512_xor_si512(z5, z30); + z11 = _mm512_xor_si512(z11, z30); + /* Rho - rotate each lane in place */ + z13 = _mm512_rol_epi64(z13, 1); + z21 = _mm512_rol_epi64(z21, 62); + z9 = _mm512_rol_epi64(z9, 28); + z17 = _mm512_rol_epi64(z17, 27); + z6 = _mm512_rol_epi64(z6, 36); + z19 = _mm512_rol_epi64(z19, 44); + z2 = _mm512_rol_epi64(z2, 6); + z10 = _mm512_rol_epi64(z10, 55); + z23 = _mm512_rol_epi64(z23, 20); + z12 = _mm512_rol_epi64(z12, 3); + z20 = _mm512_rol_epi64(z20, 10); + z8 = _mm512_rol_epi64(z8, 43); + z16 = _mm512_rol_epi64(z16, 25); + z4 = _mm512_rol_epi64(z4, 39); + z18 = _mm512_rol_epi64(z18, 41); + z1 = _mm512_rol_epi64(z1, 45); + z14 = _mm512_rol_epi64(z14, 15); + z22 = _mm512_rol_epi64(z22, 21); + z5 = _mm512_rol_epi64(z5, 8); + z24 = _mm512_rol_epi64(z24, 18); + z7 = _mm512_rol_epi64(z7, 2); + z15 = _mm512_rol_epi64(z15, 61); + z3 = _mm512_rol_epi64(z3, 56); + z11 = _mm512_rol_epi64(z11, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z19; + z0 = _mm512_ternarylogic_epi64(z0, z19, z8, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z8, z22, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z22, z11, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z9; + z31 = z23; + z9 = _mm512_ternarylogic_epi64(z9, z23, z12, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z12, z1, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z1, z15, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z13; + z31 = z2; + z13 = _mm512_ternarylogic_epi64(z13, z2, z16, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z16, z5, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z5, z24, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z17; + z31 = z6; + z17 = _mm512_ternarylogic_epi64(z17, z6, z20, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z20, z14, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z14, z3, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z21; + z31 = z10; + z21 = _mm512_ternarylogic_epi64(z21, z10, z4, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z4, z18, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z18, z7, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 512))); + /* Round 9 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z9, z13, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z17, z21, 0x96); + z26 = z19; + z26 = _mm512_ternarylogic_epi64(z26, z23, z2, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z6, z10, 0x96); + z27 = z8; + z27 = _mm512_ternarylogic_epi64(z27, z12, z16, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z20, z4, 0x96); + z28 = z22; + z28 = _mm512_ternarylogic_epi64(z28, z1, z5, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z14, z18, 0x96); + z29 = z11; + z29 = _mm512_ternarylogic_epi64(z29, z15, z24, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z3, z7, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z9 = _mm512_xor_si512(z9, z30); + z13 = _mm512_xor_si512(z13, z30); + z17 = _mm512_xor_si512(z17, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z19 = _mm512_xor_si512(z19, z30); + z23 = _mm512_xor_si512(z23, z30); + z2 = _mm512_xor_si512(z2, z30); + z6 = _mm512_xor_si512(z6, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z8 = _mm512_xor_si512(z8, z30); + z12 = _mm512_xor_si512(z12, z30); + z16 = _mm512_xor_si512(z16, z30); + z20 = _mm512_xor_si512(z20, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z22 = _mm512_xor_si512(z22, z30); + z1 = _mm512_xor_si512(z1, z30); + z5 = _mm512_xor_si512(z5, z30); + z14 = _mm512_xor_si512(z14, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z11 = _mm512_xor_si512(z11, z30); + z15 = _mm512_xor_si512(z15, z30); + z24 = _mm512_xor_si512(z24, z30); + z3 = _mm512_xor_si512(z3, z30); + z7 = _mm512_xor_si512(z7, z30); + /* Rho - rotate each lane in place */ + z19 = _mm512_rol_epi64(z19, 1); + z8 = _mm512_rol_epi64(z8, 62); + z22 = _mm512_rol_epi64(z22, 28); + z11 = _mm512_rol_epi64(z11, 27); + z9 = _mm512_rol_epi64(z9, 36); + z23 = _mm512_rol_epi64(z23, 44); + z12 = _mm512_rol_epi64(z12, 6); + z1 = _mm512_rol_epi64(z1, 55); + z15 = _mm512_rol_epi64(z15, 20); + z13 = _mm512_rol_epi64(z13, 3); + z2 = _mm512_rol_epi64(z2, 10); + z16 = _mm512_rol_epi64(z16, 43); + z5 = _mm512_rol_epi64(z5, 25); + z24 = _mm512_rol_epi64(z24, 39); + z17 = _mm512_rol_epi64(z17, 41); + z6 = _mm512_rol_epi64(z6, 45); + z20 = _mm512_rol_epi64(z20, 15); + z14 = _mm512_rol_epi64(z14, 21); + z3 = _mm512_rol_epi64(z3, 8); + z21 = _mm512_rol_epi64(z21, 18); + z10 = _mm512_rol_epi64(z10, 2); + z4 = _mm512_rol_epi64(z4, 61); + z18 = _mm512_rol_epi64(z18, 56); + z7 = _mm512_rol_epi64(z7, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z23; + z0 = _mm512_ternarylogic_epi64(z0, z23, z16, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z16, z14, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z14, z7, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z22; + z31 = z15; + z22 = _mm512_ternarylogic_epi64(z22, z15, z13, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z13, z6, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z6, z4, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z19; + z31 = z12; + z19 = _mm512_ternarylogic_epi64(z19, z12, z5, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z5, z3, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z3, z21, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z11; + z31 = z9; + z11 = _mm512_ternarylogic_epi64(z11, z9, z2, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z2, z20, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z20, z18, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z8; + z31 = z1; + z8 = _mm512_ternarylogic_epi64(z8, z1, z24, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z24, z17, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z17, z10, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 576))); + /* Round 10 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z22, z19, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z11, z8, 0x96); + z26 = z23; + z26 = _mm512_ternarylogic_epi64(z26, z15, z12, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z9, z1, 0x96); + z27 = z16; + z27 = _mm512_ternarylogic_epi64(z27, z13, z5, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z2, z24, 0x96); + z28 = z14; + z28 = _mm512_ternarylogic_epi64(z28, z6, z3, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z20, z17, 0x96); + z29 = z7; + z29 = _mm512_ternarylogic_epi64(z29, z4, z21, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z18, z10, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z22 = _mm512_xor_si512(z22, z30); + z19 = _mm512_xor_si512(z19, z30); + z11 = _mm512_xor_si512(z11, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z23 = _mm512_xor_si512(z23, z30); + z15 = _mm512_xor_si512(z15, z30); + z12 = _mm512_xor_si512(z12, z30); + z9 = _mm512_xor_si512(z9, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z16 = _mm512_xor_si512(z16, z30); + z13 = _mm512_xor_si512(z13, z30); + z5 = _mm512_xor_si512(z5, z30); + z2 = _mm512_xor_si512(z2, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z14 = _mm512_xor_si512(z14, z30); + z6 = _mm512_xor_si512(z6, z30); + z3 = _mm512_xor_si512(z3, z30); + z20 = _mm512_xor_si512(z20, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z7 = _mm512_xor_si512(z7, z30); + z4 = _mm512_xor_si512(z4, z30); + z21 = _mm512_xor_si512(z21, z30); + z18 = _mm512_xor_si512(z18, z30); + z10 = _mm512_xor_si512(z10, z30); + /* Rho - rotate each lane in place */ + z23 = _mm512_rol_epi64(z23, 1); + z16 = _mm512_rol_epi64(z16, 62); + z14 = _mm512_rol_epi64(z14, 28); + z7 = _mm512_rol_epi64(z7, 27); + z22 = _mm512_rol_epi64(z22, 36); + z15 = _mm512_rol_epi64(z15, 44); + z13 = _mm512_rol_epi64(z13, 6); + z6 = _mm512_rol_epi64(z6, 55); + z4 = _mm512_rol_epi64(z4, 20); + z19 = _mm512_rol_epi64(z19, 3); + z12 = _mm512_rol_epi64(z12, 10); + z5 = _mm512_rol_epi64(z5, 43); + z3 = _mm512_rol_epi64(z3, 25); + z21 = _mm512_rol_epi64(z21, 39); + z11 = _mm512_rol_epi64(z11, 41); + z9 = _mm512_rol_epi64(z9, 45); + z2 = _mm512_rol_epi64(z2, 15); + z20 = _mm512_rol_epi64(z20, 21); + z18 = _mm512_rol_epi64(z18, 8); + z8 = _mm512_rol_epi64(z8, 18); + z1 = _mm512_rol_epi64(z1, 2); + z24 = _mm512_rol_epi64(z24, 61); + z17 = _mm512_rol_epi64(z17, 56); + z10 = _mm512_rol_epi64(z10, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z15; + z0 = _mm512_ternarylogic_epi64(z0, z15, z5, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z5, z20, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z20, z10, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z14; + z31 = z4; + z14 = _mm512_ternarylogic_epi64(z14, z4, z19, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z19, z9, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z9, z24, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z23; + z31 = z13; + z23 = _mm512_ternarylogic_epi64(z23, z13, z3, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z3, z18, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z18, z8, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z7; + z31 = z22; + z7 = _mm512_ternarylogic_epi64(z7, z22, z12, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z12, z2, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z2, z17, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z16; + z31 = z6; + z16 = _mm512_ternarylogic_epi64(z16, z6, z21, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z21, z11, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z11, z1, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 640))); + /* Round 11 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z14, z23, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z7, z16, 0x96); + z26 = z15; + z26 = _mm512_ternarylogic_epi64(z26, z4, z13, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z22, z6, 0x96); + z27 = z5; + z27 = _mm512_ternarylogic_epi64(z27, z19, z3, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z12, z21, 0x96); + z28 = z20; + z28 = _mm512_ternarylogic_epi64(z28, z9, z18, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z2, z11, 0x96); + z29 = z10; + z29 = _mm512_ternarylogic_epi64(z29, z24, z8, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z17, z1, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z14 = _mm512_xor_si512(z14, z30); + z23 = _mm512_xor_si512(z23, z30); + z7 = _mm512_xor_si512(z7, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z15 = _mm512_xor_si512(z15, z30); + z4 = _mm512_xor_si512(z4, z30); + z13 = _mm512_xor_si512(z13, z30); + z22 = _mm512_xor_si512(z22, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z5 = _mm512_xor_si512(z5, z30); + z19 = _mm512_xor_si512(z19, z30); + z3 = _mm512_xor_si512(z3, z30); + z12 = _mm512_xor_si512(z12, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z20 = _mm512_xor_si512(z20, z30); + z9 = _mm512_xor_si512(z9, z30); + z18 = _mm512_xor_si512(z18, z30); + z2 = _mm512_xor_si512(z2, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z10 = _mm512_xor_si512(z10, z30); + z24 = _mm512_xor_si512(z24, z30); + z8 = _mm512_xor_si512(z8, z30); + z17 = _mm512_xor_si512(z17, z30); + z1 = _mm512_xor_si512(z1, z30); + /* Rho - rotate each lane in place */ + z15 = _mm512_rol_epi64(z15, 1); + z5 = _mm512_rol_epi64(z5, 62); + z20 = _mm512_rol_epi64(z20, 28); + z10 = _mm512_rol_epi64(z10, 27); + z14 = _mm512_rol_epi64(z14, 36); + z4 = _mm512_rol_epi64(z4, 44); + z19 = _mm512_rol_epi64(z19, 6); + z9 = _mm512_rol_epi64(z9, 55); + z24 = _mm512_rol_epi64(z24, 20); + z23 = _mm512_rol_epi64(z23, 3); + z13 = _mm512_rol_epi64(z13, 10); + z3 = _mm512_rol_epi64(z3, 43); + z18 = _mm512_rol_epi64(z18, 25); + z8 = _mm512_rol_epi64(z8, 39); + z7 = _mm512_rol_epi64(z7, 41); + z22 = _mm512_rol_epi64(z22, 45); + z12 = _mm512_rol_epi64(z12, 15); + z2 = _mm512_rol_epi64(z2, 21); + z17 = _mm512_rol_epi64(z17, 8); + z16 = _mm512_rol_epi64(z16, 18); + z6 = _mm512_rol_epi64(z6, 2); + z21 = _mm512_rol_epi64(z21, 61); + z11 = _mm512_rol_epi64(z11, 56); + z1 = _mm512_rol_epi64(z1, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z4; + z0 = _mm512_ternarylogic_epi64(z0, z4, z3, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z3, z2, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z2, z1, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z20; + z31 = z24; + z20 = _mm512_ternarylogic_epi64(z20, z24, z23, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z23, z22, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z22, z21, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z15; + z31 = z19; + z15 = _mm512_ternarylogic_epi64(z15, z19, z18, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z18, z17, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z17, z16, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z10; + z31 = z14; + z10 = _mm512_ternarylogic_epi64(z10, z14, z13, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z13, z12, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z12, z11, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z5; + z31 = z9; + z5 = _mm512_ternarylogic_epi64(z5, z9, z8, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z8, z7, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z7, z6, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 704))); + /* Round 12 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z20, z15, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z10, z5, 0x96); + z26 = z4; + z26 = _mm512_ternarylogic_epi64(z26, z24, z19, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z14, z9, 0x96); + z27 = z3; + z27 = _mm512_ternarylogic_epi64(z27, z23, z18, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z13, z8, 0x96); + z28 = z2; + z28 = _mm512_ternarylogic_epi64(z28, z22, z17, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z12, z7, 0x96); + z29 = z1; + z29 = _mm512_ternarylogic_epi64(z29, z21, z16, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z11, z6, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z20 = _mm512_xor_si512(z20, z30); + z15 = _mm512_xor_si512(z15, z30); + z10 = _mm512_xor_si512(z10, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z4 = _mm512_xor_si512(z4, z30); + z24 = _mm512_xor_si512(z24, z30); + z19 = _mm512_xor_si512(z19, z30); + z14 = _mm512_xor_si512(z14, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z3 = _mm512_xor_si512(z3, z30); + z23 = _mm512_xor_si512(z23, z30); + z18 = _mm512_xor_si512(z18, z30); + z13 = _mm512_xor_si512(z13, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z2 = _mm512_xor_si512(z2, z30); + z22 = _mm512_xor_si512(z22, z30); + z17 = _mm512_xor_si512(z17, z30); + z12 = _mm512_xor_si512(z12, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z1 = _mm512_xor_si512(z1, z30); + z21 = _mm512_xor_si512(z21, z30); + z16 = _mm512_xor_si512(z16, z30); + z11 = _mm512_xor_si512(z11, z30); + z6 = _mm512_xor_si512(z6, z30); + /* Rho - rotate each lane in place */ + z4 = _mm512_rol_epi64(z4, 1); + z3 = _mm512_rol_epi64(z3, 62); + z2 = _mm512_rol_epi64(z2, 28); + z1 = _mm512_rol_epi64(z1, 27); + z20 = _mm512_rol_epi64(z20, 36); + z24 = _mm512_rol_epi64(z24, 44); + z23 = _mm512_rol_epi64(z23, 6); + z22 = _mm512_rol_epi64(z22, 55); + z21 = _mm512_rol_epi64(z21, 20); + z15 = _mm512_rol_epi64(z15, 3); + z19 = _mm512_rol_epi64(z19, 10); + z18 = _mm512_rol_epi64(z18, 43); + z17 = _mm512_rol_epi64(z17, 25); + z16 = _mm512_rol_epi64(z16, 39); + z10 = _mm512_rol_epi64(z10, 41); + z14 = _mm512_rol_epi64(z14, 45); + z13 = _mm512_rol_epi64(z13, 15); + z12 = _mm512_rol_epi64(z12, 21); + z11 = _mm512_rol_epi64(z11, 8); + z5 = _mm512_rol_epi64(z5, 18); + z9 = _mm512_rol_epi64(z9, 2); + z8 = _mm512_rol_epi64(z8, 61); + z7 = _mm512_rol_epi64(z7, 56); + z6 = _mm512_rol_epi64(z6, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z24; + z0 = _mm512_ternarylogic_epi64(z0, z24, z18, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z18, z12, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z12, z6, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z2; + z31 = z21; + z2 = _mm512_ternarylogic_epi64(z2, z21, z15, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z15, z14, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z14, z8, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z4; + z31 = z23; + z4 = _mm512_ternarylogic_epi64(z4, z23, z17, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z17, z11, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z11, z5, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z1; + z31 = z20; + z1 = _mm512_ternarylogic_epi64(z1, z20, z19, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z19, z13, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z13, z7, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z3; + z31 = z22; + z3 = _mm512_ternarylogic_epi64(z3, z22, z16, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z16, z10, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z10, z9, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 768))); + /* Round 13 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z2, z4, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z1, z3, 0x96); + z26 = z24; + z26 = _mm512_ternarylogic_epi64(z26, z21, z23, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z20, z22, 0x96); + z27 = z18; + z27 = _mm512_ternarylogic_epi64(z27, z15, z17, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z19, z16, 0x96); + z28 = z12; + z28 = _mm512_ternarylogic_epi64(z28, z14, z11, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z13, z10, 0x96); + z29 = z6; + z29 = _mm512_ternarylogic_epi64(z29, z8, z5, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z7, z9, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z2 = _mm512_xor_si512(z2, z30); + z4 = _mm512_xor_si512(z4, z30); + z1 = _mm512_xor_si512(z1, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z24 = _mm512_xor_si512(z24, z30); + z21 = _mm512_xor_si512(z21, z30); + z23 = _mm512_xor_si512(z23, z30); + z20 = _mm512_xor_si512(z20, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z18 = _mm512_xor_si512(z18, z30); + z15 = _mm512_xor_si512(z15, z30); + z17 = _mm512_xor_si512(z17, z30); + z19 = _mm512_xor_si512(z19, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z12 = _mm512_xor_si512(z12, z30); + z14 = _mm512_xor_si512(z14, z30); + z11 = _mm512_xor_si512(z11, z30); + z13 = _mm512_xor_si512(z13, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z6 = _mm512_xor_si512(z6, z30); + z8 = _mm512_xor_si512(z8, z30); + z5 = _mm512_xor_si512(z5, z30); + z7 = _mm512_xor_si512(z7, z30); + z9 = _mm512_xor_si512(z9, z30); + /* Rho - rotate each lane in place */ + z24 = _mm512_rol_epi64(z24, 1); + z18 = _mm512_rol_epi64(z18, 62); + z12 = _mm512_rol_epi64(z12, 28); + z6 = _mm512_rol_epi64(z6, 27); + z2 = _mm512_rol_epi64(z2, 36); + z21 = _mm512_rol_epi64(z21, 44); + z15 = _mm512_rol_epi64(z15, 6); + z14 = _mm512_rol_epi64(z14, 55); + z8 = _mm512_rol_epi64(z8, 20); + z4 = _mm512_rol_epi64(z4, 3); + z23 = _mm512_rol_epi64(z23, 10); + z17 = _mm512_rol_epi64(z17, 43); + z11 = _mm512_rol_epi64(z11, 25); + z5 = _mm512_rol_epi64(z5, 39); + z1 = _mm512_rol_epi64(z1, 41); + z20 = _mm512_rol_epi64(z20, 45); + z19 = _mm512_rol_epi64(z19, 15); + z13 = _mm512_rol_epi64(z13, 21); + z7 = _mm512_rol_epi64(z7, 8); + z3 = _mm512_rol_epi64(z3, 18); + z22 = _mm512_rol_epi64(z22, 2); + z16 = _mm512_rol_epi64(z16, 61); + z10 = _mm512_rol_epi64(z10, 56); + z9 = _mm512_rol_epi64(z9, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z21; + z0 = _mm512_ternarylogic_epi64(z0, z21, z17, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z17, z13, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z13, z9, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z12; + z31 = z8; + z12 = _mm512_ternarylogic_epi64(z12, z8, z4, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z4, z20, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z20, z16, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z24; + z31 = z15; + z24 = _mm512_ternarylogic_epi64(z24, z15, z11, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z11, z7, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z7, z3, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z6; + z31 = z2; + z6 = _mm512_ternarylogic_epi64(z6, z2, z23, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z23, z19, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z19, z10, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z18; + z31 = z14; + z18 = _mm512_ternarylogic_epi64(z18, z14, z5, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z5, z1, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z1, z22, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 832))); + /* Round 14 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z12, z24, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z6, z18, 0x96); + z26 = z21; + z26 = _mm512_ternarylogic_epi64(z26, z8, z15, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z2, z14, 0x96); + z27 = z17; + z27 = _mm512_ternarylogic_epi64(z27, z4, z11, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z23, z5, 0x96); + z28 = z13; + z28 = _mm512_ternarylogic_epi64(z28, z20, z7, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z19, z1, 0x96); + z29 = z9; + z29 = _mm512_ternarylogic_epi64(z29, z16, z3, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z10, z22, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z12 = _mm512_xor_si512(z12, z30); + z24 = _mm512_xor_si512(z24, z30); + z6 = _mm512_xor_si512(z6, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z21 = _mm512_xor_si512(z21, z30); + z8 = _mm512_xor_si512(z8, z30); + z15 = _mm512_xor_si512(z15, z30); + z2 = _mm512_xor_si512(z2, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z17 = _mm512_xor_si512(z17, z30); + z4 = _mm512_xor_si512(z4, z30); + z11 = _mm512_xor_si512(z11, z30); + z23 = _mm512_xor_si512(z23, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z13 = _mm512_xor_si512(z13, z30); + z20 = _mm512_xor_si512(z20, z30); + z7 = _mm512_xor_si512(z7, z30); + z19 = _mm512_xor_si512(z19, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z9 = _mm512_xor_si512(z9, z30); + z16 = _mm512_xor_si512(z16, z30); + z3 = _mm512_xor_si512(z3, z30); + z10 = _mm512_xor_si512(z10, z30); + z22 = _mm512_xor_si512(z22, z30); + /* Rho - rotate each lane in place */ + z21 = _mm512_rol_epi64(z21, 1); + z17 = _mm512_rol_epi64(z17, 62); + z13 = _mm512_rol_epi64(z13, 28); + z9 = _mm512_rol_epi64(z9, 27); + z12 = _mm512_rol_epi64(z12, 36); + z8 = _mm512_rol_epi64(z8, 44); + z4 = _mm512_rol_epi64(z4, 6); + z20 = _mm512_rol_epi64(z20, 55); + z16 = _mm512_rol_epi64(z16, 20); + z24 = _mm512_rol_epi64(z24, 3); + z15 = _mm512_rol_epi64(z15, 10); + z11 = _mm512_rol_epi64(z11, 43); + z7 = _mm512_rol_epi64(z7, 25); + z3 = _mm512_rol_epi64(z3, 39); + z6 = _mm512_rol_epi64(z6, 41); + z2 = _mm512_rol_epi64(z2, 45); + z23 = _mm512_rol_epi64(z23, 15); + z19 = _mm512_rol_epi64(z19, 21); + z10 = _mm512_rol_epi64(z10, 8); + z18 = _mm512_rol_epi64(z18, 18); + z14 = _mm512_rol_epi64(z14, 2); + z5 = _mm512_rol_epi64(z5, 61); + z1 = _mm512_rol_epi64(z1, 56); + z22 = _mm512_rol_epi64(z22, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z8; + z0 = _mm512_ternarylogic_epi64(z0, z8, z11, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z11, z19, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z19, z22, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z13; + z31 = z16; + z13 = _mm512_ternarylogic_epi64(z13, z16, z24, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z24, z2, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z2, z5, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z21; + z31 = z4; + z21 = _mm512_ternarylogic_epi64(z21, z4, z7, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z7, z10, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z10, z18, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z9; + z31 = z12; + z9 = _mm512_ternarylogic_epi64(z9, z12, z15, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z15, z23, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z23, z1, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z17; + z31 = z20; + z17 = _mm512_ternarylogic_epi64(z17, z20, z3, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z3, z6, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z6, z14, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 896))); + /* Round 15 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z13, z21, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z9, z17, 0x96); + z26 = z8; + z26 = _mm512_ternarylogic_epi64(z26, z16, z4, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z12, z20, 0x96); + z27 = z11; + z27 = _mm512_ternarylogic_epi64(z27, z24, z7, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z15, z3, 0x96); + z28 = z19; + z28 = _mm512_ternarylogic_epi64(z28, z2, z10, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z23, z6, 0x96); + z29 = z22; + z29 = _mm512_ternarylogic_epi64(z29, z5, z18, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z1, z14, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z13 = _mm512_xor_si512(z13, z30); + z21 = _mm512_xor_si512(z21, z30); + z9 = _mm512_xor_si512(z9, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z8 = _mm512_xor_si512(z8, z30); + z16 = _mm512_xor_si512(z16, z30); + z4 = _mm512_xor_si512(z4, z30); + z12 = _mm512_xor_si512(z12, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z11 = _mm512_xor_si512(z11, z30); + z24 = _mm512_xor_si512(z24, z30); + z7 = _mm512_xor_si512(z7, z30); + z15 = _mm512_xor_si512(z15, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z19 = _mm512_xor_si512(z19, z30); + z2 = _mm512_xor_si512(z2, z30); + z10 = _mm512_xor_si512(z10, z30); + z23 = _mm512_xor_si512(z23, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z22 = _mm512_xor_si512(z22, z30); + z5 = _mm512_xor_si512(z5, z30); + z18 = _mm512_xor_si512(z18, z30); + z1 = _mm512_xor_si512(z1, z30); + z14 = _mm512_xor_si512(z14, z30); + /* Rho - rotate each lane in place */ + z8 = _mm512_rol_epi64(z8, 1); + z11 = _mm512_rol_epi64(z11, 62); + z19 = _mm512_rol_epi64(z19, 28); + z22 = _mm512_rol_epi64(z22, 27); + z13 = _mm512_rol_epi64(z13, 36); + z16 = _mm512_rol_epi64(z16, 44); + z24 = _mm512_rol_epi64(z24, 6); + z2 = _mm512_rol_epi64(z2, 55); + z5 = _mm512_rol_epi64(z5, 20); + z21 = _mm512_rol_epi64(z21, 3); + z4 = _mm512_rol_epi64(z4, 10); + z7 = _mm512_rol_epi64(z7, 43); + z10 = _mm512_rol_epi64(z10, 25); + z18 = _mm512_rol_epi64(z18, 39); + z9 = _mm512_rol_epi64(z9, 41); + z12 = _mm512_rol_epi64(z12, 45); + z15 = _mm512_rol_epi64(z15, 15); + z23 = _mm512_rol_epi64(z23, 21); + z1 = _mm512_rol_epi64(z1, 8); + z17 = _mm512_rol_epi64(z17, 18); + z20 = _mm512_rol_epi64(z20, 2); + z3 = _mm512_rol_epi64(z3, 61); + z6 = _mm512_rol_epi64(z6, 56); + z14 = _mm512_rol_epi64(z14, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z16; + z0 = _mm512_ternarylogic_epi64(z0, z16, z7, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z7, z23, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z23, z14, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z19; + z31 = z5; + z19 = _mm512_ternarylogic_epi64(z19, z5, z21, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z21, z12, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z12, z3, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z8; + z31 = z24; + z8 = _mm512_ternarylogic_epi64(z8, z24, z10, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z10, z1, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z1, z17, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z22; + z31 = z13; + z22 = _mm512_ternarylogic_epi64(z22, z13, z4, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z4, z15, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z15, z6, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z11; + z31 = z2; + z11 = _mm512_ternarylogic_epi64(z11, z2, z18, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z18, z9, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z9, z20, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 960))); + /* Round 16 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z19, z8, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z22, z11, 0x96); + z26 = z16; + z26 = _mm512_ternarylogic_epi64(z26, z5, z24, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z13, z2, 0x96); + z27 = z7; + z27 = _mm512_ternarylogic_epi64(z27, z21, z10, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z4, z18, 0x96); + z28 = z23; + z28 = _mm512_ternarylogic_epi64(z28, z12, z1, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z15, z9, 0x96); + z29 = z14; + z29 = _mm512_ternarylogic_epi64(z29, z3, z17, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z6, z20, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z19 = _mm512_xor_si512(z19, z30); + z8 = _mm512_xor_si512(z8, z30); + z22 = _mm512_xor_si512(z22, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z16 = _mm512_xor_si512(z16, z30); + z5 = _mm512_xor_si512(z5, z30); + z24 = _mm512_xor_si512(z24, z30); + z13 = _mm512_xor_si512(z13, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z7 = _mm512_xor_si512(z7, z30); + z21 = _mm512_xor_si512(z21, z30); + z10 = _mm512_xor_si512(z10, z30); + z4 = _mm512_xor_si512(z4, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z23 = _mm512_xor_si512(z23, z30); + z12 = _mm512_xor_si512(z12, z30); + z1 = _mm512_xor_si512(z1, z30); + z15 = _mm512_xor_si512(z15, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z14 = _mm512_xor_si512(z14, z30); + z3 = _mm512_xor_si512(z3, z30); + z17 = _mm512_xor_si512(z17, z30); + z6 = _mm512_xor_si512(z6, z30); + z20 = _mm512_xor_si512(z20, z30); + /* Rho - rotate each lane in place */ + z16 = _mm512_rol_epi64(z16, 1); + z7 = _mm512_rol_epi64(z7, 62); + z23 = _mm512_rol_epi64(z23, 28); + z14 = _mm512_rol_epi64(z14, 27); + z19 = _mm512_rol_epi64(z19, 36); + z5 = _mm512_rol_epi64(z5, 44); + z21 = _mm512_rol_epi64(z21, 6); + z12 = _mm512_rol_epi64(z12, 55); + z3 = _mm512_rol_epi64(z3, 20); + z8 = _mm512_rol_epi64(z8, 3); + z24 = _mm512_rol_epi64(z24, 10); + z10 = _mm512_rol_epi64(z10, 43); + z1 = _mm512_rol_epi64(z1, 25); + z17 = _mm512_rol_epi64(z17, 39); + z22 = _mm512_rol_epi64(z22, 41); + z13 = _mm512_rol_epi64(z13, 45); + z4 = _mm512_rol_epi64(z4, 15); + z15 = _mm512_rol_epi64(z15, 21); + z6 = _mm512_rol_epi64(z6, 8); + z11 = _mm512_rol_epi64(z11, 18); + z2 = _mm512_rol_epi64(z2, 2); + z18 = _mm512_rol_epi64(z18, 61); + z9 = _mm512_rol_epi64(z9, 56); + z20 = _mm512_rol_epi64(z20, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z5; + z0 = _mm512_ternarylogic_epi64(z0, z5, z10, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z10, z15, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z15, z20, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z23; + z31 = z3; + z23 = _mm512_ternarylogic_epi64(z23, z3, z8, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z8, z13, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z13, z18, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z16; + z31 = z21; + z16 = _mm512_ternarylogic_epi64(z16, z21, z1, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z1, z6, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z6, z11, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z14; + z31 = z19; + z14 = _mm512_ternarylogic_epi64(z14, z19, z24, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z24, z4, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z4, z9, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z7; + z31 = z12; + z7 = _mm512_ternarylogic_epi64(z7, z12, z17, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z17, z22, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z22, z2, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1024))); + /* Round 17 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z23, z16, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z14, z7, 0x96); + z26 = z5; + z26 = _mm512_ternarylogic_epi64(z26, z3, z21, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z19, z12, 0x96); + z27 = z10; + z27 = _mm512_ternarylogic_epi64(z27, z8, z1, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z24, z17, 0x96); + z28 = z15; + z28 = _mm512_ternarylogic_epi64(z28, z13, z6, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z4, z22, 0x96); + z29 = z20; + z29 = _mm512_ternarylogic_epi64(z29, z18, z11, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z9, z2, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z23 = _mm512_xor_si512(z23, z30); + z16 = _mm512_xor_si512(z16, z30); + z14 = _mm512_xor_si512(z14, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z5 = _mm512_xor_si512(z5, z30); + z3 = _mm512_xor_si512(z3, z30); + z21 = _mm512_xor_si512(z21, z30); + z19 = _mm512_xor_si512(z19, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z10 = _mm512_xor_si512(z10, z30); + z8 = _mm512_xor_si512(z8, z30); + z1 = _mm512_xor_si512(z1, z30); + z24 = _mm512_xor_si512(z24, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z15 = _mm512_xor_si512(z15, z30); + z13 = _mm512_xor_si512(z13, z30); + z6 = _mm512_xor_si512(z6, z30); + z4 = _mm512_xor_si512(z4, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z20 = _mm512_xor_si512(z20, z30); + z18 = _mm512_xor_si512(z18, z30); + z11 = _mm512_xor_si512(z11, z30); + z9 = _mm512_xor_si512(z9, z30); + z2 = _mm512_xor_si512(z2, z30); + /* Rho - rotate each lane in place */ + z5 = _mm512_rol_epi64(z5, 1); + z10 = _mm512_rol_epi64(z10, 62); + z15 = _mm512_rol_epi64(z15, 28); + z20 = _mm512_rol_epi64(z20, 27); + z23 = _mm512_rol_epi64(z23, 36); + z3 = _mm512_rol_epi64(z3, 44); + z8 = _mm512_rol_epi64(z8, 6); + z13 = _mm512_rol_epi64(z13, 55); + z18 = _mm512_rol_epi64(z18, 20); + z16 = _mm512_rol_epi64(z16, 3); + z21 = _mm512_rol_epi64(z21, 10); + z1 = _mm512_rol_epi64(z1, 43); + z6 = _mm512_rol_epi64(z6, 25); + z11 = _mm512_rol_epi64(z11, 39); + z14 = _mm512_rol_epi64(z14, 41); + z19 = _mm512_rol_epi64(z19, 45); + z24 = _mm512_rol_epi64(z24, 15); + z4 = _mm512_rol_epi64(z4, 21); + z9 = _mm512_rol_epi64(z9, 8); + z7 = _mm512_rol_epi64(z7, 18); + z12 = _mm512_rol_epi64(z12, 2); + z17 = _mm512_rol_epi64(z17, 61); + z22 = _mm512_rol_epi64(z22, 56); + z2 = _mm512_rol_epi64(z2, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z3; + z0 = _mm512_ternarylogic_epi64(z0, z3, z1, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z1, z4, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z4, z2, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z15; + z31 = z18; + z15 = _mm512_ternarylogic_epi64(z15, z18, z16, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z16, z19, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z19, z17, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z5; + z31 = z8; + z5 = _mm512_ternarylogic_epi64(z5, z8, z6, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z6, z9, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z9, z7, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z20; + z31 = z23; + z20 = _mm512_ternarylogic_epi64(z20, z23, z21, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z21, z24, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z24, z22, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z10; + z31 = z13; + z10 = _mm512_ternarylogic_epi64(z10, z13, z11, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z11, z14, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z14, z12, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1088))); + /* Round 18 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z15, z5, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z20, z10, 0x96); + z26 = z3; + z26 = _mm512_ternarylogic_epi64(z26, z18, z8, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z23, z13, 0x96); + z27 = z1; + z27 = _mm512_ternarylogic_epi64(z27, z16, z6, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z21, z11, 0x96); + z28 = z4; + z28 = _mm512_ternarylogic_epi64(z28, z19, z9, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z24, z14, 0x96); + z29 = z2; + z29 = _mm512_ternarylogic_epi64(z29, z17, z7, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z22, z12, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z15 = _mm512_xor_si512(z15, z30); + z5 = _mm512_xor_si512(z5, z30); + z20 = _mm512_xor_si512(z20, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z3 = _mm512_xor_si512(z3, z30); + z18 = _mm512_xor_si512(z18, z30); + z8 = _mm512_xor_si512(z8, z30); + z23 = _mm512_xor_si512(z23, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z1 = _mm512_xor_si512(z1, z30); + z16 = _mm512_xor_si512(z16, z30); + z6 = _mm512_xor_si512(z6, z30); + z21 = _mm512_xor_si512(z21, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z4 = _mm512_xor_si512(z4, z30); + z19 = _mm512_xor_si512(z19, z30); + z9 = _mm512_xor_si512(z9, z30); + z24 = _mm512_xor_si512(z24, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z2 = _mm512_xor_si512(z2, z30); + z17 = _mm512_xor_si512(z17, z30); + z7 = _mm512_xor_si512(z7, z30); + z22 = _mm512_xor_si512(z22, z30); + z12 = _mm512_xor_si512(z12, z30); + /* Rho - rotate each lane in place */ + z3 = _mm512_rol_epi64(z3, 1); + z1 = _mm512_rol_epi64(z1, 62); + z4 = _mm512_rol_epi64(z4, 28); + z2 = _mm512_rol_epi64(z2, 27); + z15 = _mm512_rol_epi64(z15, 36); + z18 = _mm512_rol_epi64(z18, 44); + z16 = _mm512_rol_epi64(z16, 6); + z19 = _mm512_rol_epi64(z19, 55); + z17 = _mm512_rol_epi64(z17, 20); + z5 = _mm512_rol_epi64(z5, 3); + z8 = _mm512_rol_epi64(z8, 10); + z6 = _mm512_rol_epi64(z6, 43); + z9 = _mm512_rol_epi64(z9, 25); + z7 = _mm512_rol_epi64(z7, 39); + z20 = _mm512_rol_epi64(z20, 41); + z23 = _mm512_rol_epi64(z23, 45); + z21 = _mm512_rol_epi64(z21, 15); + z24 = _mm512_rol_epi64(z24, 21); + z22 = _mm512_rol_epi64(z22, 8); + z10 = _mm512_rol_epi64(z10, 18); + z13 = _mm512_rol_epi64(z13, 2); + z11 = _mm512_rol_epi64(z11, 61); + z14 = _mm512_rol_epi64(z14, 56); + z12 = _mm512_rol_epi64(z12, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z18; + z0 = _mm512_ternarylogic_epi64(z0, z18, z6, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z6, z24, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z24, z12, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z4; + z31 = z17; + z4 = _mm512_ternarylogic_epi64(z4, z17, z5, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z5, z23, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z23, z11, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z3; + z31 = z16; + z3 = _mm512_ternarylogic_epi64(z3, z16, z9, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z9, z22, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z22, z10, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z2; + z31 = z15; + z2 = _mm512_ternarylogic_epi64(z2, z15, z8, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z8, z21, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z21, z14, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z1; + z31 = z19; + z1 = _mm512_ternarylogic_epi64(z1, z19, z7, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z7, z20, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z20, z13, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1152))); + /* Round 19 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z4, z3, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z2, z1, 0x96); + z26 = z18; + z26 = _mm512_ternarylogic_epi64(z26, z17, z16, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z15, z19, 0x96); + z27 = z6; + z27 = _mm512_ternarylogic_epi64(z27, z5, z9, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z8, z7, 0x96); + z28 = z24; + z28 = _mm512_ternarylogic_epi64(z28, z23, z22, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z21, z20, 0x96); + z29 = z12; + z29 = _mm512_ternarylogic_epi64(z29, z11, z10, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z14, z13, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z4 = _mm512_xor_si512(z4, z30); + z3 = _mm512_xor_si512(z3, z30); + z2 = _mm512_xor_si512(z2, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z18 = _mm512_xor_si512(z18, z30); + z17 = _mm512_xor_si512(z17, z30); + z16 = _mm512_xor_si512(z16, z30); + z15 = _mm512_xor_si512(z15, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z6 = _mm512_xor_si512(z6, z30); + z5 = _mm512_xor_si512(z5, z30); + z9 = _mm512_xor_si512(z9, z30); + z8 = _mm512_xor_si512(z8, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z24 = _mm512_xor_si512(z24, z30); + z23 = _mm512_xor_si512(z23, z30); + z22 = _mm512_xor_si512(z22, z30); + z21 = _mm512_xor_si512(z21, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z12 = _mm512_xor_si512(z12, z30); + z11 = _mm512_xor_si512(z11, z30); + z10 = _mm512_xor_si512(z10, z30); + z14 = _mm512_xor_si512(z14, z30); + z13 = _mm512_xor_si512(z13, z30); + /* Rho - rotate each lane in place */ + z18 = _mm512_rol_epi64(z18, 1); + z6 = _mm512_rol_epi64(z6, 62); + z24 = _mm512_rol_epi64(z24, 28); + z12 = _mm512_rol_epi64(z12, 27); + z4 = _mm512_rol_epi64(z4, 36); + z17 = _mm512_rol_epi64(z17, 44); + z5 = _mm512_rol_epi64(z5, 6); + z23 = _mm512_rol_epi64(z23, 55); + z11 = _mm512_rol_epi64(z11, 20); + z3 = _mm512_rol_epi64(z3, 3); + z16 = _mm512_rol_epi64(z16, 10); + z9 = _mm512_rol_epi64(z9, 43); + z22 = _mm512_rol_epi64(z22, 25); + z10 = _mm512_rol_epi64(z10, 39); + z2 = _mm512_rol_epi64(z2, 41); + z15 = _mm512_rol_epi64(z15, 45); + z8 = _mm512_rol_epi64(z8, 15); + z21 = _mm512_rol_epi64(z21, 21); + z14 = _mm512_rol_epi64(z14, 8); + z1 = _mm512_rol_epi64(z1, 18); + z19 = _mm512_rol_epi64(z19, 2); + z7 = _mm512_rol_epi64(z7, 61); + z20 = _mm512_rol_epi64(z20, 56); + z13 = _mm512_rol_epi64(z13, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z17; + z0 = _mm512_ternarylogic_epi64(z0, z17, z9, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z9, z21, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z21, z13, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z24; + z31 = z11; + z24 = _mm512_ternarylogic_epi64(z24, z11, z3, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z3, z15, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z15, z7, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z18; + z31 = z5; + z18 = _mm512_ternarylogic_epi64(z18, z5, z22, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z22, z14, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z14, z1, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z12; + z31 = z4; + z12 = _mm512_ternarylogic_epi64(z12, z4, z16, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z16, z8, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z8, z20, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z6; + z31 = z23; + z6 = _mm512_ternarylogic_epi64(z6, z23, z10, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z10, z2, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z2, z19, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1216))); + /* Round 20 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z24, z18, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z12, z6, 0x96); + z26 = z17; + z26 = _mm512_ternarylogic_epi64(z26, z11, z5, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z4, z23, 0x96); + z27 = z9; + z27 = _mm512_ternarylogic_epi64(z27, z3, z22, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z16, z10, 0x96); + z28 = z21; + z28 = _mm512_ternarylogic_epi64(z28, z15, z14, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z8, z2, 0x96); + z29 = z13; + z29 = _mm512_ternarylogic_epi64(z29, z7, z1, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z20, z19, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z24 = _mm512_xor_si512(z24, z30); + z18 = _mm512_xor_si512(z18, z30); + z12 = _mm512_xor_si512(z12, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z17 = _mm512_xor_si512(z17, z30); + z11 = _mm512_xor_si512(z11, z30); + z5 = _mm512_xor_si512(z5, z30); + z4 = _mm512_xor_si512(z4, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z9 = _mm512_xor_si512(z9, z30); + z3 = _mm512_xor_si512(z3, z30); + z22 = _mm512_xor_si512(z22, z30); + z16 = _mm512_xor_si512(z16, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z21 = _mm512_xor_si512(z21, z30); + z15 = _mm512_xor_si512(z15, z30); + z14 = _mm512_xor_si512(z14, z30); + z8 = _mm512_xor_si512(z8, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z13 = _mm512_xor_si512(z13, z30); + z7 = _mm512_xor_si512(z7, z30); + z1 = _mm512_xor_si512(z1, z30); + z20 = _mm512_xor_si512(z20, z30); + z19 = _mm512_xor_si512(z19, z30); + /* Rho - rotate each lane in place */ + z17 = _mm512_rol_epi64(z17, 1); + z9 = _mm512_rol_epi64(z9, 62); + z21 = _mm512_rol_epi64(z21, 28); + z13 = _mm512_rol_epi64(z13, 27); + z24 = _mm512_rol_epi64(z24, 36); + z11 = _mm512_rol_epi64(z11, 44); + z3 = _mm512_rol_epi64(z3, 6); + z15 = _mm512_rol_epi64(z15, 55); + z7 = _mm512_rol_epi64(z7, 20); + z18 = _mm512_rol_epi64(z18, 3); + z5 = _mm512_rol_epi64(z5, 10); + z22 = _mm512_rol_epi64(z22, 43); + z14 = _mm512_rol_epi64(z14, 25); + z1 = _mm512_rol_epi64(z1, 39); + z12 = _mm512_rol_epi64(z12, 41); + z4 = _mm512_rol_epi64(z4, 45); + z16 = _mm512_rol_epi64(z16, 15); + z8 = _mm512_rol_epi64(z8, 21); + z20 = _mm512_rol_epi64(z20, 8); + z6 = _mm512_rol_epi64(z6, 18); + z23 = _mm512_rol_epi64(z23, 2); + z10 = _mm512_rol_epi64(z10, 61); + z2 = _mm512_rol_epi64(z2, 56); + z19 = _mm512_rol_epi64(z19, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z11; + z0 = _mm512_ternarylogic_epi64(z0, z11, z22, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z22, z8, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z8, z19, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z21; + z31 = z7; + z21 = _mm512_ternarylogic_epi64(z21, z7, z18, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z18, z4, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z4, z10, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z17; + z31 = z3; + z17 = _mm512_ternarylogic_epi64(z17, z3, z14, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z14, z20, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z20, z6, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z13; + z31 = z24; + z13 = _mm512_ternarylogic_epi64(z13, z24, z5, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z5, z16, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z16, z2, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z9; + z31 = z15; + z9 = _mm512_ternarylogic_epi64(z9, z15, z1, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z1, z12, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z12, z23, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1280))); + /* Round 21 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z21, z17, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z13, z9, 0x96); + z26 = z11; + z26 = _mm512_ternarylogic_epi64(z26, z7, z3, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z24, z15, 0x96); + z27 = z22; + z27 = _mm512_ternarylogic_epi64(z27, z18, z14, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z5, z1, 0x96); + z28 = z8; + z28 = _mm512_ternarylogic_epi64(z28, z4, z20, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z16, z12, 0x96); + z29 = z19; + z29 = _mm512_ternarylogic_epi64(z29, z10, z6, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z2, z23, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z21 = _mm512_xor_si512(z21, z30); + z17 = _mm512_xor_si512(z17, z30); + z13 = _mm512_xor_si512(z13, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z11 = _mm512_xor_si512(z11, z30); + z7 = _mm512_xor_si512(z7, z30); + z3 = _mm512_xor_si512(z3, z30); + z24 = _mm512_xor_si512(z24, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z22 = _mm512_xor_si512(z22, z30); + z18 = _mm512_xor_si512(z18, z30); + z14 = _mm512_xor_si512(z14, z30); + z5 = _mm512_xor_si512(z5, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z8 = _mm512_xor_si512(z8, z30); + z4 = _mm512_xor_si512(z4, z30); + z20 = _mm512_xor_si512(z20, z30); + z16 = _mm512_xor_si512(z16, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z19 = _mm512_xor_si512(z19, z30); + z10 = _mm512_xor_si512(z10, z30); + z6 = _mm512_xor_si512(z6, z30); + z2 = _mm512_xor_si512(z2, z30); + z23 = _mm512_xor_si512(z23, z30); + /* Rho - rotate each lane in place */ + z11 = _mm512_rol_epi64(z11, 1); + z22 = _mm512_rol_epi64(z22, 62); + z8 = _mm512_rol_epi64(z8, 28); + z19 = _mm512_rol_epi64(z19, 27); + z21 = _mm512_rol_epi64(z21, 36); + z7 = _mm512_rol_epi64(z7, 44); + z18 = _mm512_rol_epi64(z18, 6); + z4 = _mm512_rol_epi64(z4, 55); + z10 = _mm512_rol_epi64(z10, 20); + z17 = _mm512_rol_epi64(z17, 3); + z3 = _mm512_rol_epi64(z3, 10); + z14 = _mm512_rol_epi64(z14, 43); + z20 = _mm512_rol_epi64(z20, 25); + z6 = _mm512_rol_epi64(z6, 39); + z13 = _mm512_rol_epi64(z13, 41); + z24 = _mm512_rol_epi64(z24, 45); + z5 = _mm512_rol_epi64(z5, 15); + z16 = _mm512_rol_epi64(z16, 21); + z2 = _mm512_rol_epi64(z2, 8); + z9 = _mm512_rol_epi64(z9, 18); + z15 = _mm512_rol_epi64(z15, 2); + z1 = _mm512_rol_epi64(z1, 61); + z12 = _mm512_rol_epi64(z12, 56); + z23 = _mm512_rol_epi64(z23, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z7; + z0 = _mm512_ternarylogic_epi64(z0, z7, z14, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z14, z16, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z16, z23, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z8; + z31 = z10; + z8 = _mm512_ternarylogic_epi64(z8, z10, z17, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z17, z24, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z24, z1, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z11; + z31 = z18; + z11 = _mm512_ternarylogic_epi64(z11, z18, z20, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z20, z2, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z2, z9, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z19; + z31 = z21; + z19 = _mm512_ternarylogic_epi64(z19, z21, z3, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z3, z5, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z5, z12, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z22; + z31 = z4; + z22 = _mm512_ternarylogic_epi64(z22, z4, z6, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z6, z13, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z13, z15, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1344))); + /* Round 22 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z8, z11, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z19, z22, 0x96); + z26 = z7; + z26 = _mm512_ternarylogic_epi64(z26, z10, z18, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z21, z4, 0x96); + z27 = z14; + z27 = _mm512_ternarylogic_epi64(z27, z17, z20, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z3, z6, 0x96); + z28 = z16; + z28 = _mm512_ternarylogic_epi64(z28, z24, z2, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z5, z13, 0x96); + z29 = z23; + z29 = _mm512_ternarylogic_epi64(z29, z1, z9, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z12, z15, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z8 = _mm512_xor_si512(z8, z30); + z11 = _mm512_xor_si512(z11, z30); + z19 = _mm512_xor_si512(z19, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z7 = _mm512_xor_si512(z7, z30); + z10 = _mm512_xor_si512(z10, z30); + z18 = _mm512_xor_si512(z18, z30); + z21 = _mm512_xor_si512(z21, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z14 = _mm512_xor_si512(z14, z30); + z17 = _mm512_xor_si512(z17, z30); + z20 = _mm512_xor_si512(z20, z30); + z3 = _mm512_xor_si512(z3, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z16 = _mm512_xor_si512(z16, z30); + z24 = _mm512_xor_si512(z24, z30); + z2 = _mm512_xor_si512(z2, z30); + z5 = _mm512_xor_si512(z5, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z23 = _mm512_xor_si512(z23, z30); + z1 = _mm512_xor_si512(z1, z30); + z9 = _mm512_xor_si512(z9, z30); + z12 = _mm512_xor_si512(z12, z30); + z15 = _mm512_xor_si512(z15, z30); + /* Rho - rotate each lane in place */ + z7 = _mm512_rol_epi64(z7, 1); + z14 = _mm512_rol_epi64(z14, 62); + z16 = _mm512_rol_epi64(z16, 28); + z23 = _mm512_rol_epi64(z23, 27); + z8 = _mm512_rol_epi64(z8, 36); + z10 = _mm512_rol_epi64(z10, 44); + z17 = _mm512_rol_epi64(z17, 6); + z24 = _mm512_rol_epi64(z24, 55); + z1 = _mm512_rol_epi64(z1, 20); + z11 = _mm512_rol_epi64(z11, 3); + z18 = _mm512_rol_epi64(z18, 10); + z20 = _mm512_rol_epi64(z20, 43); + z2 = _mm512_rol_epi64(z2, 25); + z9 = _mm512_rol_epi64(z9, 39); + z19 = _mm512_rol_epi64(z19, 41); + z21 = _mm512_rol_epi64(z21, 45); + z3 = _mm512_rol_epi64(z3, 15); + z5 = _mm512_rol_epi64(z5, 21); + z12 = _mm512_rol_epi64(z12, 8); + z22 = _mm512_rol_epi64(z22, 18); + z4 = _mm512_rol_epi64(z4, 2); + z6 = _mm512_rol_epi64(z6, 61); + z13 = _mm512_rol_epi64(z13, 56); + z15 = _mm512_rol_epi64(z15, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z10; + z0 = _mm512_ternarylogic_epi64(z0, z10, z20, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z20, z5, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z5, z15, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z16; + z31 = z1; + z16 = _mm512_ternarylogic_epi64(z16, z1, z11, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z11, z21, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z21, z6, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z7; + z31 = z17; + z7 = _mm512_ternarylogic_epi64(z7, z17, z2, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z2, z12, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z12, z22, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z23; + z31 = z8; + z23 = _mm512_ternarylogic_epi64(z23, z8, z18, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z18, z3, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z3, z13, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z14; + z31 = z24; + z14 = _mm512_ternarylogic_epi64(z14, z24, z9, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z9, z19, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z19, z4, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1408))); + /* Round 23 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z16, z7, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z23, z14, 0x96); + z26 = z10; + z26 = _mm512_ternarylogic_epi64(z26, z1, z17, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z8, z24, 0x96); + z27 = z20; + z27 = _mm512_ternarylogic_epi64(z27, z11, z2, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z18, z9, 0x96); + z28 = z5; + z28 = _mm512_ternarylogic_epi64(z28, z21, z12, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z3, z19, 0x96); + z29 = z15; + z29 = _mm512_ternarylogic_epi64(z29, z6, z22, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z13, z4, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z16 = _mm512_xor_si512(z16, z30); + z7 = _mm512_xor_si512(z7, z30); + z23 = _mm512_xor_si512(z23, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z10 = _mm512_xor_si512(z10, z30); + z1 = _mm512_xor_si512(z1, z30); + z17 = _mm512_xor_si512(z17, z30); + z8 = _mm512_xor_si512(z8, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z20 = _mm512_xor_si512(z20, z30); + z11 = _mm512_xor_si512(z11, z30); + z2 = _mm512_xor_si512(z2, z30); + z18 = _mm512_xor_si512(z18, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z5 = _mm512_xor_si512(z5, z30); + z21 = _mm512_xor_si512(z21, z30); + z12 = _mm512_xor_si512(z12, z30); + z3 = _mm512_xor_si512(z3, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z15 = _mm512_xor_si512(z15, z30); + z6 = _mm512_xor_si512(z6, z30); + z22 = _mm512_xor_si512(z22, z30); + z13 = _mm512_xor_si512(z13, z30); + z4 = _mm512_xor_si512(z4, z30); + /* Rho - rotate each lane in place */ + z10 = _mm512_rol_epi64(z10, 1); + z20 = _mm512_rol_epi64(z20, 62); + z5 = _mm512_rol_epi64(z5, 28); + z15 = _mm512_rol_epi64(z15, 27); + z16 = _mm512_rol_epi64(z16, 36); + z1 = _mm512_rol_epi64(z1, 44); + z11 = _mm512_rol_epi64(z11, 6); + z21 = _mm512_rol_epi64(z21, 55); + z6 = _mm512_rol_epi64(z6, 20); + z7 = _mm512_rol_epi64(z7, 3); + z17 = _mm512_rol_epi64(z17, 10); + z2 = _mm512_rol_epi64(z2, 43); + z12 = _mm512_rol_epi64(z12, 25); + z22 = _mm512_rol_epi64(z22, 39); + z23 = _mm512_rol_epi64(z23, 41); + z8 = _mm512_rol_epi64(z8, 45); + z18 = _mm512_rol_epi64(z18, 15); + z3 = _mm512_rol_epi64(z3, 21); + z13 = _mm512_rol_epi64(z13, 8); + z14 = _mm512_rol_epi64(z14, 18); + z24 = _mm512_rol_epi64(z24, 2); + z9 = _mm512_rol_epi64(z9, 61); + z19 = _mm512_rol_epi64(z19, 56); + z4 = _mm512_rol_epi64(z4, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z1; + z0 = _mm512_ternarylogic_epi64(z0, z1, z2, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z2, z3, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z3, z4, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z5; + z31 = z6; + z5 = _mm512_ternarylogic_epi64(z5, z6, z7, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z7, z8, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z8, z9, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z10; + z31 = z11; + z10 = _mm512_ternarylogic_epi64(z10, z11, z12, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z12, z13, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z13, z14, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z15; + z31 = z16; + z15 = _mm512_ternarylogic_epi64(z15, z16, z17, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z17, z18, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z18, z19, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z20; + z31 = z21; + z20 = _mm512_ternarylogic_epi64(z20, z21, z22, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z22, z23, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z23, z24, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1472))); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 1024), z16); + _mm512_storeu_si512((void*)WC_PW(rdi, 1088), z17); + _mm512_storeu_si512((void*)WC_PW(rdi, 1152), z18); + _mm512_storeu_si512((void*)WC_PW(rdi, 1216), z19); + _mm512_storeu_si512((void*)WC_PW(rdi, 1280), z20); + _mm512_storeu_si512((void*)WC_PW(rdi, 1344), z21); + _mm512_storeu_si512((void*)WC_PW(rdi, 1408), z22); + _mm512_storeu_si512((void*)WC_PW(rdi, 1472), z23); + _mm512_storeu_si512((void*)WC_PW(rdi, 1536), z24); +} + +#endif /* WOLFSSL_HAVE_MLKEM */ +#ifdef WOLFSSL_HAVE_MLDSA +XALIGNED(32) static const word64 L_sha3_256_blocksx8_seed_64_avx512_end_mark[] + WC_X64I_UNUSED = { + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, + 0x8000000000000000ULL, 0x8000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void sha3_256_blocksx8_seed_64_avx512(word64* s, byte* seed) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23, z24, z25, z26, z27, + z28, z29, z30, z31; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)seed; + + rdx = (word64)(L_sha3_x8_avx512_r); + z0 = _mm512_set1_epi64((long long)WC_L64(rsi, 0)); + z1 = _mm512_set1_epi64((long long)WC_L64(rsi, 8)); + z2 = _mm512_set1_epi64((long long)WC_L64(rsi, 16)); + z3 = _mm512_set1_epi64((long long)WC_L64(rsi, 24)); + z4 = _mm512_set1_epi64((long long)WC_L64(rsi, 32)); + z5 = _mm512_set1_epi64((long long)WC_L64(rsi, 40)); + z6 = _mm512_set1_epi64((long long)WC_L64(rsi, 48)); + z7 = _mm512_set1_epi64((long long)WC_L64(rsi, 56)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z9 = _mm512_setzero_si512(); + z10 = z9; + z11 = z9; + z12 = z9; + z13 = z9; + z14 = z9; + z15 = z9; + z16 = _mm512_loadu_si512((const void*)WC_PR( + L_sha3_256_blocksx8_seed_64_avx512_end_mark, 0)); + z17 = z9; + z18 = z9; + z19 = z9; + z20 = z9; + z21 = z9; + z22 = z9; + z23 = z9; + z24 = z9; + /* Round 0 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z5, z10, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z15, z20, 0x96); + z26 = z1; + z26 = _mm512_ternarylogic_epi64(z26, z6, z11, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z16, z21, 0x96); + z27 = z2; + z27 = _mm512_ternarylogic_epi64(z27, z7, z12, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z17, z22, 0x96); + z28 = z3; + z28 = _mm512_ternarylogic_epi64(z28, z8, z13, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z18, z23, 0x96); + z29 = z4; + z29 = _mm512_ternarylogic_epi64(z29, z9, z14, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z19, z24, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z5 = _mm512_xor_si512(z5, z30); + z10 = _mm512_xor_si512(z10, z30); + z15 = _mm512_xor_si512(z15, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z1 = _mm512_xor_si512(z1, z30); + z6 = _mm512_xor_si512(z6, z30); + z11 = _mm512_xor_si512(z11, z30); + z16 = _mm512_xor_si512(z16, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z2 = _mm512_xor_si512(z2, z30); + z7 = _mm512_xor_si512(z7, z30); + z12 = _mm512_xor_si512(z12, z30); + z17 = _mm512_xor_si512(z17, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z3 = _mm512_xor_si512(z3, z30); + z8 = _mm512_xor_si512(z8, z30); + z13 = _mm512_xor_si512(z13, z30); + z18 = _mm512_xor_si512(z18, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z4 = _mm512_xor_si512(z4, z30); + z9 = _mm512_xor_si512(z9, z30); + z14 = _mm512_xor_si512(z14, z30); + z19 = _mm512_xor_si512(z19, z30); + z24 = _mm512_xor_si512(z24, z30); + /* Rho - rotate each lane in place */ + z1 = _mm512_rol_epi64(z1, 1); + z2 = _mm512_rol_epi64(z2, 62); + z3 = _mm512_rol_epi64(z3, 28); + z4 = _mm512_rol_epi64(z4, 27); + z5 = _mm512_rol_epi64(z5, 36); + z6 = _mm512_rol_epi64(z6, 44); + z7 = _mm512_rol_epi64(z7, 6); + z8 = _mm512_rol_epi64(z8, 55); + z9 = _mm512_rol_epi64(z9, 20); + z10 = _mm512_rol_epi64(z10, 3); + z11 = _mm512_rol_epi64(z11, 10); + z12 = _mm512_rol_epi64(z12, 43); + z13 = _mm512_rol_epi64(z13, 25); + z14 = _mm512_rol_epi64(z14, 39); + z15 = _mm512_rol_epi64(z15, 41); + z16 = _mm512_rol_epi64(z16, 45); + z17 = _mm512_rol_epi64(z17, 15); + z18 = _mm512_rol_epi64(z18, 21); + z19 = _mm512_rol_epi64(z19, 8); + z20 = _mm512_rol_epi64(z20, 18); + z21 = _mm512_rol_epi64(z21, 2); + z22 = _mm512_rol_epi64(z22, 61); + z23 = _mm512_rol_epi64(z23, 56); + z24 = _mm512_rol_epi64(z24, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z6; + z0 = _mm512_ternarylogic_epi64(z0, z6, z12, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z12, z18, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z18, z24, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z3; + z31 = z9; + z3 = _mm512_ternarylogic_epi64(z3, z9, z10, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z10, z16, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z16, z22, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z1; + z31 = z7; + z1 = _mm512_ternarylogic_epi64(z1, z7, z13, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z13, z19, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z19, z20, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z4; + z31 = z5; + z4 = _mm512_ternarylogic_epi64(z4, z5, z11, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z11, z17, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z17, z23, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z2; + z31 = z8; + z2 = _mm512_ternarylogic_epi64(z2, z8, z14, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z14, z15, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z15, z21, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 0))); + /* Round 1 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z3, z1, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z4, z2, 0x96); + z26 = z6; + z26 = _mm512_ternarylogic_epi64(z26, z9, z7, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z5, z8, 0x96); + z27 = z12; + z27 = _mm512_ternarylogic_epi64(z27, z10, z13, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z11, z14, 0x96); + z28 = z18; + z28 = _mm512_ternarylogic_epi64(z28, z16, z19, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z17, z15, 0x96); + z29 = z24; + z29 = _mm512_ternarylogic_epi64(z29, z22, z20, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z23, z21, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z3 = _mm512_xor_si512(z3, z30); + z1 = _mm512_xor_si512(z1, z30); + z4 = _mm512_xor_si512(z4, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z6 = _mm512_xor_si512(z6, z30); + z9 = _mm512_xor_si512(z9, z30); + z7 = _mm512_xor_si512(z7, z30); + z5 = _mm512_xor_si512(z5, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z12 = _mm512_xor_si512(z12, z30); + z10 = _mm512_xor_si512(z10, z30); + z13 = _mm512_xor_si512(z13, z30); + z11 = _mm512_xor_si512(z11, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z18 = _mm512_xor_si512(z18, z30); + z16 = _mm512_xor_si512(z16, z30); + z19 = _mm512_xor_si512(z19, z30); + z17 = _mm512_xor_si512(z17, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z24 = _mm512_xor_si512(z24, z30); + z22 = _mm512_xor_si512(z22, z30); + z20 = _mm512_xor_si512(z20, z30); + z23 = _mm512_xor_si512(z23, z30); + z21 = _mm512_xor_si512(z21, z30); + /* Rho - rotate each lane in place */ + z6 = _mm512_rol_epi64(z6, 1); + z12 = _mm512_rol_epi64(z12, 62); + z18 = _mm512_rol_epi64(z18, 28); + z24 = _mm512_rol_epi64(z24, 27); + z3 = _mm512_rol_epi64(z3, 36); + z9 = _mm512_rol_epi64(z9, 44); + z10 = _mm512_rol_epi64(z10, 6); + z16 = _mm512_rol_epi64(z16, 55); + z22 = _mm512_rol_epi64(z22, 20); + z1 = _mm512_rol_epi64(z1, 3); + z7 = _mm512_rol_epi64(z7, 10); + z13 = _mm512_rol_epi64(z13, 43); + z19 = _mm512_rol_epi64(z19, 25); + z20 = _mm512_rol_epi64(z20, 39); + z4 = _mm512_rol_epi64(z4, 41); + z5 = _mm512_rol_epi64(z5, 45); + z11 = _mm512_rol_epi64(z11, 15); + z17 = _mm512_rol_epi64(z17, 21); + z23 = _mm512_rol_epi64(z23, 8); + z2 = _mm512_rol_epi64(z2, 18); + z8 = _mm512_rol_epi64(z8, 2); + z14 = _mm512_rol_epi64(z14, 61); + z15 = _mm512_rol_epi64(z15, 56); + z21 = _mm512_rol_epi64(z21, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z9; + z0 = _mm512_ternarylogic_epi64(z0, z9, z13, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z13, z17, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z17, z21, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z18; + z31 = z22; + z18 = _mm512_ternarylogic_epi64(z18, z22, z1, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z1, z5, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z5, z14, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z6; + z31 = z10; + z6 = _mm512_ternarylogic_epi64(z6, z10, z19, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z19, z23, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z23, z2, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z24; + z31 = z3; + z24 = _mm512_ternarylogic_epi64(z24, z3, z7, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z7, z11, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z11, z15, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z12; + z31 = z16; + z12 = _mm512_ternarylogic_epi64(z12, z16, z20, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z20, z4, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z4, z8, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 64))); + /* Round 2 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z18, z6, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z24, z12, 0x96); + z26 = z9; + z26 = _mm512_ternarylogic_epi64(z26, z22, z10, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z3, z16, 0x96); + z27 = z13; + z27 = _mm512_ternarylogic_epi64(z27, z1, z19, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z7, z20, 0x96); + z28 = z17; + z28 = _mm512_ternarylogic_epi64(z28, z5, z23, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z11, z4, 0x96); + z29 = z21; + z29 = _mm512_ternarylogic_epi64(z29, z14, z2, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z15, z8, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z18 = _mm512_xor_si512(z18, z30); + z6 = _mm512_xor_si512(z6, z30); + z24 = _mm512_xor_si512(z24, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z9 = _mm512_xor_si512(z9, z30); + z22 = _mm512_xor_si512(z22, z30); + z10 = _mm512_xor_si512(z10, z30); + z3 = _mm512_xor_si512(z3, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z13 = _mm512_xor_si512(z13, z30); + z1 = _mm512_xor_si512(z1, z30); + z19 = _mm512_xor_si512(z19, z30); + z7 = _mm512_xor_si512(z7, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z17 = _mm512_xor_si512(z17, z30); + z5 = _mm512_xor_si512(z5, z30); + z23 = _mm512_xor_si512(z23, z30); + z11 = _mm512_xor_si512(z11, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z21 = _mm512_xor_si512(z21, z30); + z14 = _mm512_xor_si512(z14, z30); + z2 = _mm512_xor_si512(z2, z30); + z15 = _mm512_xor_si512(z15, z30); + z8 = _mm512_xor_si512(z8, z30); + /* Rho - rotate each lane in place */ + z9 = _mm512_rol_epi64(z9, 1); + z13 = _mm512_rol_epi64(z13, 62); + z17 = _mm512_rol_epi64(z17, 28); + z21 = _mm512_rol_epi64(z21, 27); + z18 = _mm512_rol_epi64(z18, 36); + z22 = _mm512_rol_epi64(z22, 44); + z1 = _mm512_rol_epi64(z1, 6); + z5 = _mm512_rol_epi64(z5, 55); + z14 = _mm512_rol_epi64(z14, 20); + z6 = _mm512_rol_epi64(z6, 3); + z10 = _mm512_rol_epi64(z10, 10); + z19 = _mm512_rol_epi64(z19, 43); + z23 = _mm512_rol_epi64(z23, 25); + z2 = _mm512_rol_epi64(z2, 39); + z24 = _mm512_rol_epi64(z24, 41); + z3 = _mm512_rol_epi64(z3, 45); + z7 = _mm512_rol_epi64(z7, 15); + z11 = _mm512_rol_epi64(z11, 21); + z15 = _mm512_rol_epi64(z15, 8); + z12 = _mm512_rol_epi64(z12, 18); + z16 = _mm512_rol_epi64(z16, 2); + z20 = _mm512_rol_epi64(z20, 61); + z4 = _mm512_rol_epi64(z4, 56); + z8 = _mm512_rol_epi64(z8, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z22; + z0 = _mm512_ternarylogic_epi64(z0, z22, z19, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z19, z11, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z11, z8, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z17; + z31 = z14; + z17 = _mm512_ternarylogic_epi64(z17, z14, z6, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z6, z3, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z3, z20, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z9; + z31 = z1; + z9 = _mm512_ternarylogic_epi64(z9, z1, z23, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z23, z15, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z15, z12, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z21; + z31 = z18; + z21 = _mm512_ternarylogic_epi64(z21, z18, z10, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z10, z7, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z7, z4, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z13; + z31 = z5; + z13 = _mm512_ternarylogic_epi64(z13, z5, z2, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z2, z24, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z24, z16, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 128))); + /* Round 3 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z17, z9, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z21, z13, 0x96); + z26 = z22; + z26 = _mm512_ternarylogic_epi64(z26, z14, z1, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z18, z5, 0x96); + z27 = z19; + z27 = _mm512_ternarylogic_epi64(z27, z6, z23, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z10, z2, 0x96); + z28 = z11; + z28 = _mm512_ternarylogic_epi64(z28, z3, z15, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z7, z24, 0x96); + z29 = z8; + z29 = _mm512_ternarylogic_epi64(z29, z20, z12, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z4, z16, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z17 = _mm512_xor_si512(z17, z30); + z9 = _mm512_xor_si512(z9, z30); + z21 = _mm512_xor_si512(z21, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z22 = _mm512_xor_si512(z22, z30); + z14 = _mm512_xor_si512(z14, z30); + z1 = _mm512_xor_si512(z1, z30); + z18 = _mm512_xor_si512(z18, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z19 = _mm512_xor_si512(z19, z30); + z6 = _mm512_xor_si512(z6, z30); + z23 = _mm512_xor_si512(z23, z30); + z10 = _mm512_xor_si512(z10, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z11 = _mm512_xor_si512(z11, z30); + z3 = _mm512_xor_si512(z3, z30); + z15 = _mm512_xor_si512(z15, z30); + z7 = _mm512_xor_si512(z7, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z8 = _mm512_xor_si512(z8, z30); + z20 = _mm512_xor_si512(z20, z30); + z12 = _mm512_xor_si512(z12, z30); + z4 = _mm512_xor_si512(z4, z30); + z16 = _mm512_xor_si512(z16, z30); + /* Rho - rotate each lane in place */ + z22 = _mm512_rol_epi64(z22, 1); + z19 = _mm512_rol_epi64(z19, 62); + z11 = _mm512_rol_epi64(z11, 28); + z8 = _mm512_rol_epi64(z8, 27); + z17 = _mm512_rol_epi64(z17, 36); + z14 = _mm512_rol_epi64(z14, 44); + z6 = _mm512_rol_epi64(z6, 6); + z3 = _mm512_rol_epi64(z3, 55); + z20 = _mm512_rol_epi64(z20, 20); + z9 = _mm512_rol_epi64(z9, 3); + z1 = _mm512_rol_epi64(z1, 10); + z23 = _mm512_rol_epi64(z23, 43); + z15 = _mm512_rol_epi64(z15, 25); + z12 = _mm512_rol_epi64(z12, 39); + z21 = _mm512_rol_epi64(z21, 41); + z18 = _mm512_rol_epi64(z18, 45); + z10 = _mm512_rol_epi64(z10, 15); + z7 = _mm512_rol_epi64(z7, 21); + z4 = _mm512_rol_epi64(z4, 8); + z13 = _mm512_rol_epi64(z13, 18); + z5 = _mm512_rol_epi64(z5, 2); + z2 = _mm512_rol_epi64(z2, 61); + z24 = _mm512_rol_epi64(z24, 56); + z16 = _mm512_rol_epi64(z16, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z14; + z0 = _mm512_ternarylogic_epi64(z0, z14, z23, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z23, z7, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z7, z16, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z11; + z31 = z20; + z11 = _mm512_ternarylogic_epi64(z11, z20, z9, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z9, z18, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z18, z2, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z22; + z31 = z6; + z22 = _mm512_ternarylogic_epi64(z22, z6, z15, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z15, z4, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z4, z13, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z8; + z31 = z17; + z8 = _mm512_ternarylogic_epi64(z8, z17, z1, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z1, z10, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z10, z24, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z19; + z31 = z3; + z19 = _mm512_ternarylogic_epi64(z19, z3, z12, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z12, z21, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z21, z5, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 192))); + /* Round 4 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z11, z22, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z8, z19, 0x96); + z26 = z14; + z26 = _mm512_ternarylogic_epi64(z26, z20, z6, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z17, z3, 0x96); + z27 = z23; + z27 = _mm512_ternarylogic_epi64(z27, z9, z15, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z1, z12, 0x96); + z28 = z7; + z28 = _mm512_ternarylogic_epi64(z28, z18, z4, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z10, z21, 0x96); + z29 = z16; + z29 = _mm512_ternarylogic_epi64(z29, z2, z13, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z24, z5, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z11 = _mm512_xor_si512(z11, z30); + z22 = _mm512_xor_si512(z22, z30); + z8 = _mm512_xor_si512(z8, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z14 = _mm512_xor_si512(z14, z30); + z20 = _mm512_xor_si512(z20, z30); + z6 = _mm512_xor_si512(z6, z30); + z17 = _mm512_xor_si512(z17, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z23 = _mm512_xor_si512(z23, z30); + z9 = _mm512_xor_si512(z9, z30); + z15 = _mm512_xor_si512(z15, z30); + z1 = _mm512_xor_si512(z1, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z7 = _mm512_xor_si512(z7, z30); + z18 = _mm512_xor_si512(z18, z30); + z4 = _mm512_xor_si512(z4, z30); + z10 = _mm512_xor_si512(z10, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z16 = _mm512_xor_si512(z16, z30); + z2 = _mm512_xor_si512(z2, z30); + z13 = _mm512_xor_si512(z13, z30); + z24 = _mm512_xor_si512(z24, z30); + z5 = _mm512_xor_si512(z5, z30); + /* Rho - rotate each lane in place */ + z14 = _mm512_rol_epi64(z14, 1); + z23 = _mm512_rol_epi64(z23, 62); + z7 = _mm512_rol_epi64(z7, 28); + z16 = _mm512_rol_epi64(z16, 27); + z11 = _mm512_rol_epi64(z11, 36); + z20 = _mm512_rol_epi64(z20, 44); + z9 = _mm512_rol_epi64(z9, 6); + z18 = _mm512_rol_epi64(z18, 55); + z2 = _mm512_rol_epi64(z2, 20); + z22 = _mm512_rol_epi64(z22, 3); + z6 = _mm512_rol_epi64(z6, 10); + z15 = _mm512_rol_epi64(z15, 43); + z4 = _mm512_rol_epi64(z4, 25); + z13 = _mm512_rol_epi64(z13, 39); + z8 = _mm512_rol_epi64(z8, 41); + z17 = _mm512_rol_epi64(z17, 45); + z1 = _mm512_rol_epi64(z1, 15); + z10 = _mm512_rol_epi64(z10, 21); + z24 = _mm512_rol_epi64(z24, 8); + z19 = _mm512_rol_epi64(z19, 18); + z3 = _mm512_rol_epi64(z3, 2); + z12 = _mm512_rol_epi64(z12, 61); + z21 = _mm512_rol_epi64(z21, 56); + z5 = _mm512_rol_epi64(z5, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z20; + z0 = _mm512_ternarylogic_epi64(z0, z20, z15, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z15, z10, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z10, z5, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z7; + z31 = z2; + z7 = _mm512_ternarylogic_epi64(z7, z2, z22, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z22, z17, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z17, z12, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z14; + z31 = z9; + z14 = _mm512_ternarylogic_epi64(z14, z9, z4, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z4, z24, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z24, z19, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z16; + z31 = z11; + z16 = _mm512_ternarylogic_epi64(z16, z11, z6, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z6, z1, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z1, z21, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z23; + z31 = z18; + z23 = _mm512_ternarylogic_epi64(z23, z18, z13, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z13, z8, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z8, z3, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 256))); + /* Round 5 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z7, z14, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z16, z23, 0x96); + z26 = z20; + z26 = _mm512_ternarylogic_epi64(z26, z2, z9, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z11, z18, 0x96); + z27 = z15; + z27 = _mm512_ternarylogic_epi64(z27, z22, z4, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z6, z13, 0x96); + z28 = z10; + z28 = _mm512_ternarylogic_epi64(z28, z17, z24, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z1, z8, 0x96); + z29 = z5; + z29 = _mm512_ternarylogic_epi64(z29, z12, z19, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z21, z3, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z7 = _mm512_xor_si512(z7, z30); + z14 = _mm512_xor_si512(z14, z30); + z16 = _mm512_xor_si512(z16, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z20 = _mm512_xor_si512(z20, z30); + z2 = _mm512_xor_si512(z2, z30); + z9 = _mm512_xor_si512(z9, z30); + z11 = _mm512_xor_si512(z11, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z15 = _mm512_xor_si512(z15, z30); + z22 = _mm512_xor_si512(z22, z30); + z4 = _mm512_xor_si512(z4, z30); + z6 = _mm512_xor_si512(z6, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z10 = _mm512_xor_si512(z10, z30); + z17 = _mm512_xor_si512(z17, z30); + z24 = _mm512_xor_si512(z24, z30); + z1 = _mm512_xor_si512(z1, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z5 = _mm512_xor_si512(z5, z30); + z12 = _mm512_xor_si512(z12, z30); + z19 = _mm512_xor_si512(z19, z30); + z21 = _mm512_xor_si512(z21, z30); + z3 = _mm512_xor_si512(z3, z30); + /* Rho - rotate each lane in place */ + z20 = _mm512_rol_epi64(z20, 1); + z15 = _mm512_rol_epi64(z15, 62); + z10 = _mm512_rol_epi64(z10, 28); + z5 = _mm512_rol_epi64(z5, 27); + z7 = _mm512_rol_epi64(z7, 36); + z2 = _mm512_rol_epi64(z2, 44); + z22 = _mm512_rol_epi64(z22, 6); + z17 = _mm512_rol_epi64(z17, 55); + z12 = _mm512_rol_epi64(z12, 20); + z14 = _mm512_rol_epi64(z14, 3); + z9 = _mm512_rol_epi64(z9, 10); + z4 = _mm512_rol_epi64(z4, 43); + z24 = _mm512_rol_epi64(z24, 25); + z19 = _mm512_rol_epi64(z19, 39); + z16 = _mm512_rol_epi64(z16, 41); + z11 = _mm512_rol_epi64(z11, 45); + z6 = _mm512_rol_epi64(z6, 15); + z1 = _mm512_rol_epi64(z1, 21); + z21 = _mm512_rol_epi64(z21, 8); + z23 = _mm512_rol_epi64(z23, 18); + z18 = _mm512_rol_epi64(z18, 2); + z13 = _mm512_rol_epi64(z13, 61); + z8 = _mm512_rol_epi64(z8, 56); + z3 = _mm512_rol_epi64(z3, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z2; + z0 = _mm512_ternarylogic_epi64(z0, z2, z4, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z4, z1, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z1, z3, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z10; + z31 = z12; + z10 = _mm512_ternarylogic_epi64(z10, z12, z14, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z14, z11, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z11, z13, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z20; + z31 = z22; + z20 = _mm512_ternarylogic_epi64(z20, z22, z24, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z24, z21, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z21, z23, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z5; + z31 = z7; + z5 = _mm512_ternarylogic_epi64(z5, z7, z9, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z9, z6, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z6, z8, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z15; + z31 = z17; + z15 = _mm512_ternarylogic_epi64(z15, z17, z19, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z19, z16, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z16, z18, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 320))); + /* Round 6 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z10, z20, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z5, z15, 0x96); + z26 = z2; + z26 = _mm512_ternarylogic_epi64(z26, z12, z22, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z7, z17, 0x96); + z27 = z4; + z27 = _mm512_ternarylogic_epi64(z27, z14, z24, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z9, z19, 0x96); + z28 = z1; + z28 = _mm512_ternarylogic_epi64(z28, z11, z21, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z6, z16, 0x96); + z29 = z3; + z29 = _mm512_ternarylogic_epi64(z29, z13, z23, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z8, z18, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z10 = _mm512_xor_si512(z10, z30); + z20 = _mm512_xor_si512(z20, z30); + z5 = _mm512_xor_si512(z5, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z2 = _mm512_xor_si512(z2, z30); + z12 = _mm512_xor_si512(z12, z30); + z22 = _mm512_xor_si512(z22, z30); + z7 = _mm512_xor_si512(z7, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z4 = _mm512_xor_si512(z4, z30); + z14 = _mm512_xor_si512(z14, z30); + z24 = _mm512_xor_si512(z24, z30); + z9 = _mm512_xor_si512(z9, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z1 = _mm512_xor_si512(z1, z30); + z11 = _mm512_xor_si512(z11, z30); + z21 = _mm512_xor_si512(z21, z30); + z6 = _mm512_xor_si512(z6, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z3 = _mm512_xor_si512(z3, z30); + z13 = _mm512_xor_si512(z13, z30); + z23 = _mm512_xor_si512(z23, z30); + z8 = _mm512_xor_si512(z8, z30); + z18 = _mm512_xor_si512(z18, z30); + /* Rho - rotate each lane in place */ + z2 = _mm512_rol_epi64(z2, 1); + z4 = _mm512_rol_epi64(z4, 62); + z1 = _mm512_rol_epi64(z1, 28); + z3 = _mm512_rol_epi64(z3, 27); + z10 = _mm512_rol_epi64(z10, 36); + z12 = _mm512_rol_epi64(z12, 44); + z14 = _mm512_rol_epi64(z14, 6); + z11 = _mm512_rol_epi64(z11, 55); + z13 = _mm512_rol_epi64(z13, 20); + z20 = _mm512_rol_epi64(z20, 3); + z22 = _mm512_rol_epi64(z22, 10); + z24 = _mm512_rol_epi64(z24, 43); + z21 = _mm512_rol_epi64(z21, 25); + z23 = _mm512_rol_epi64(z23, 39); + z5 = _mm512_rol_epi64(z5, 41); + z7 = _mm512_rol_epi64(z7, 45); + z9 = _mm512_rol_epi64(z9, 15); + z6 = _mm512_rol_epi64(z6, 21); + z8 = _mm512_rol_epi64(z8, 8); + z15 = _mm512_rol_epi64(z15, 18); + z17 = _mm512_rol_epi64(z17, 2); + z19 = _mm512_rol_epi64(z19, 61); + z16 = _mm512_rol_epi64(z16, 56); + z18 = _mm512_rol_epi64(z18, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z12; + z0 = _mm512_ternarylogic_epi64(z0, z12, z24, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z24, z6, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z6, z18, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z1; + z31 = z13; + z1 = _mm512_ternarylogic_epi64(z1, z13, z20, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z20, z7, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z7, z19, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z2; + z31 = z14; + z2 = _mm512_ternarylogic_epi64(z2, z14, z21, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z21, z8, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z8, z15, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z3; + z31 = z10; + z3 = _mm512_ternarylogic_epi64(z3, z10, z22, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z22, z9, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z9, z16, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z4; + z31 = z11; + z4 = _mm512_ternarylogic_epi64(z4, z11, z23, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z23, z5, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z5, z17, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 384))); + /* Round 7 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z1, z2, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z3, z4, 0x96); + z26 = z12; + z26 = _mm512_ternarylogic_epi64(z26, z13, z14, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z10, z11, 0x96); + z27 = z24; + z27 = _mm512_ternarylogic_epi64(z27, z20, z21, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z22, z23, 0x96); + z28 = z6; + z28 = _mm512_ternarylogic_epi64(z28, z7, z8, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z9, z5, 0x96); + z29 = z18; + z29 = _mm512_ternarylogic_epi64(z29, z19, z15, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z16, z17, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z1 = _mm512_xor_si512(z1, z30); + z2 = _mm512_xor_si512(z2, z30); + z3 = _mm512_xor_si512(z3, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z12 = _mm512_xor_si512(z12, z30); + z13 = _mm512_xor_si512(z13, z30); + z14 = _mm512_xor_si512(z14, z30); + z10 = _mm512_xor_si512(z10, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z24 = _mm512_xor_si512(z24, z30); + z20 = _mm512_xor_si512(z20, z30); + z21 = _mm512_xor_si512(z21, z30); + z22 = _mm512_xor_si512(z22, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z6 = _mm512_xor_si512(z6, z30); + z7 = _mm512_xor_si512(z7, z30); + z8 = _mm512_xor_si512(z8, z30); + z9 = _mm512_xor_si512(z9, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z18 = _mm512_xor_si512(z18, z30); + z19 = _mm512_xor_si512(z19, z30); + z15 = _mm512_xor_si512(z15, z30); + z16 = _mm512_xor_si512(z16, z30); + z17 = _mm512_xor_si512(z17, z30); + /* Rho - rotate each lane in place */ + z12 = _mm512_rol_epi64(z12, 1); + z24 = _mm512_rol_epi64(z24, 62); + z6 = _mm512_rol_epi64(z6, 28); + z18 = _mm512_rol_epi64(z18, 27); + z1 = _mm512_rol_epi64(z1, 36); + z13 = _mm512_rol_epi64(z13, 44); + z20 = _mm512_rol_epi64(z20, 6); + z7 = _mm512_rol_epi64(z7, 55); + z19 = _mm512_rol_epi64(z19, 20); + z2 = _mm512_rol_epi64(z2, 3); + z14 = _mm512_rol_epi64(z14, 10); + z21 = _mm512_rol_epi64(z21, 43); + z8 = _mm512_rol_epi64(z8, 25); + z15 = _mm512_rol_epi64(z15, 39); + z3 = _mm512_rol_epi64(z3, 41); + z10 = _mm512_rol_epi64(z10, 45); + z22 = _mm512_rol_epi64(z22, 15); + z9 = _mm512_rol_epi64(z9, 21); + z16 = _mm512_rol_epi64(z16, 8); + z4 = _mm512_rol_epi64(z4, 18); + z11 = _mm512_rol_epi64(z11, 2); + z23 = _mm512_rol_epi64(z23, 61); + z5 = _mm512_rol_epi64(z5, 56); + z17 = _mm512_rol_epi64(z17, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z13; + z0 = _mm512_ternarylogic_epi64(z0, z13, z21, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z21, z9, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z9, z17, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z6; + z31 = z19; + z6 = _mm512_ternarylogic_epi64(z6, z19, z2, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z2, z10, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z10, z23, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z12; + z31 = z20; + z12 = _mm512_ternarylogic_epi64(z12, z20, z8, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z8, z16, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z16, z4, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z18; + z31 = z1; + z18 = _mm512_ternarylogic_epi64(z18, z1, z14, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z14, z22, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z22, z5, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z24; + z31 = z7; + z24 = _mm512_ternarylogic_epi64(z24, z7, z15, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z15, z3, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z3, z11, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 448))); + /* Round 8 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z6, z12, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z18, z24, 0x96); + z26 = z13; + z26 = _mm512_ternarylogic_epi64(z26, z19, z20, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z1, z7, 0x96); + z27 = z21; + z27 = _mm512_ternarylogic_epi64(z27, z2, z8, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z14, z15, 0x96); + z28 = z9; + z28 = _mm512_ternarylogic_epi64(z28, z10, z16, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z22, z3, 0x96); + z29 = z17; + z29 = _mm512_ternarylogic_epi64(z29, z23, z4, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z5, z11, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z6 = _mm512_xor_si512(z6, z30); + z12 = _mm512_xor_si512(z12, z30); + z18 = _mm512_xor_si512(z18, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z13 = _mm512_xor_si512(z13, z30); + z19 = _mm512_xor_si512(z19, z30); + z20 = _mm512_xor_si512(z20, z30); + z1 = _mm512_xor_si512(z1, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z21 = _mm512_xor_si512(z21, z30); + z2 = _mm512_xor_si512(z2, z30); + z8 = _mm512_xor_si512(z8, z30); + z14 = _mm512_xor_si512(z14, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z9 = _mm512_xor_si512(z9, z30); + z10 = _mm512_xor_si512(z10, z30); + z16 = _mm512_xor_si512(z16, z30); + z22 = _mm512_xor_si512(z22, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z17 = _mm512_xor_si512(z17, z30); + z23 = _mm512_xor_si512(z23, z30); + z4 = _mm512_xor_si512(z4, z30); + z5 = _mm512_xor_si512(z5, z30); + z11 = _mm512_xor_si512(z11, z30); + /* Rho - rotate each lane in place */ + z13 = _mm512_rol_epi64(z13, 1); + z21 = _mm512_rol_epi64(z21, 62); + z9 = _mm512_rol_epi64(z9, 28); + z17 = _mm512_rol_epi64(z17, 27); + z6 = _mm512_rol_epi64(z6, 36); + z19 = _mm512_rol_epi64(z19, 44); + z2 = _mm512_rol_epi64(z2, 6); + z10 = _mm512_rol_epi64(z10, 55); + z23 = _mm512_rol_epi64(z23, 20); + z12 = _mm512_rol_epi64(z12, 3); + z20 = _mm512_rol_epi64(z20, 10); + z8 = _mm512_rol_epi64(z8, 43); + z16 = _mm512_rol_epi64(z16, 25); + z4 = _mm512_rol_epi64(z4, 39); + z18 = _mm512_rol_epi64(z18, 41); + z1 = _mm512_rol_epi64(z1, 45); + z14 = _mm512_rol_epi64(z14, 15); + z22 = _mm512_rol_epi64(z22, 21); + z5 = _mm512_rol_epi64(z5, 8); + z24 = _mm512_rol_epi64(z24, 18); + z7 = _mm512_rol_epi64(z7, 2); + z15 = _mm512_rol_epi64(z15, 61); + z3 = _mm512_rol_epi64(z3, 56); + z11 = _mm512_rol_epi64(z11, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z19; + z0 = _mm512_ternarylogic_epi64(z0, z19, z8, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z8, z22, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z22, z11, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z9; + z31 = z23; + z9 = _mm512_ternarylogic_epi64(z9, z23, z12, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z12, z1, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z1, z15, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z13; + z31 = z2; + z13 = _mm512_ternarylogic_epi64(z13, z2, z16, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z16, z5, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z5, z24, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z17; + z31 = z6; + z17 = _mm512_ternarylogic_epi64(z17, z6, z20, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z20, z14, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z14, z3, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z21; + z31 = z10; + z21 = _mm512_ternarylogic_epi64(z21, z10, z4, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z4, z18, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z18, z7, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 512))); + /* Round 9 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z9, z13, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z17, z21, 0x96); + z26 = z19; + z26 = _mm512_ternarylogic_epi64(z26, z23, z2, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z6, z10, 0x96); + z27 = z8; + z27 = _mm512_ternarylogic_epi64(z27, z12, z16, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z20, z4, 0x96); + z28 = z22; + z28 = _mm512_ternarylogic_epi64(z28, z1, z5, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z14, z18, 0x96); + z29 = z11; + z29 = _mm512_ternarylogic_epi64(z29, z15, z24, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z3, z7, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z9 = _mm512_xor_si512(z9, z30); + z13 = _mm512_xor_si512(z13, z30); + z17 = _mm512_xor_si512(z17, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z19 = _mm512_xor_si512(z19, z30); + z23 = _mm512_xor_si512(z23, z30); + z2 = _mm512_xor_si512(z2, z30); + z6 = _mm512_xor_si512(z6, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z8 = _mm512_xor_si512(z8, z30); + z12 = _mm512_xor_si512(z12, z30); + z16 = _mm512_xor_si512(z16, z30); + z20 = _mm512_xor_si512(z20, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z22 = _mm512_xor_si512(z22, z30); + z1 = _mm512_xor_si512(z1, z30); + z5 = _mm512_xor_si512(z5, z30); + z14 = _mm512_xor_si512(z14, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z11 = _mm512_xor_si512(z11, z30); + z15 = _mm512_xor_si512(z15, z30); + z24 = _mm512_xor_si512(z24, z30); + z3 = _mm512_xor_si512(z3, z30); + z7 = _mm512_xor_si512(z7, z30); + /* Rho - rotate each lane in place */ + z19 = _mm512_rol_epi64(z19, 1); + z8 = _mm512_rol_epi64(z8, 62); + z22 = _mm512_rol_epi64(z22, 28); + z11 = _mm512_rol_epi64(z11, 27); + z9 = _mm512_rol_epi64(z9, 36); + z23 = _mm512_rol_epi64(z23, 44); + z12 = _mm512_rol_epi64(z12, 6); + z1 = _mm512_rol_epi64(z1, 55); + z15 = _mm512_rol_epi64(z15, 20); + z13 = _mm512_rol_epi64(z13, 3); + z2 = _mm512_rol_epi64(z2, 10); + z16 = _mm512_rol_epi64(z16, 43); + z5 = _mm512_rol_epi64(z5, 25); + z24 = _mm512_rol_epi64(z24, 39); + z17 = _mm512_rol_epi64(z17, 41); + z6 = _mm512_rol_epi64(z6, 45); + z20 = _mm512_rol_epi64(z20, 15); + z14 = _mm512_rol_epi64(z14, 21); + z3 = _mm512_rol_epi64(z3, 8); + z21 = _mm512_rol_epi64(z21, 18); + z10 = _mm512_rol_epi64(z10, 2); + z4 = _mm512_rol_epi64(z4, 61); + z18 = _mm512_rol_epi64(z18, 56); + z7 = _mm512_rol_epi64(z7, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z23; + z0 = _mm512_ternarylogic_epi64(z0, z23, z16, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z16, z14, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z14, z7, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z22; + z31 = z15; + z22 = _mm512_ternarylogic_epi64(z22, z15, z13, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z13, z6, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z6, z4, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z19; + z31 = z12; + z19 = _mm512_ternarylogic_epi64(z19, z12, z5, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z5, z3, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z3, z21, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z11; + z31 = z9; + z11 = _mm512_ternarylogic_epi64(z11, z9, z2, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z2, z20, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z20, z18, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z8; + z31 = z1; + z8 = _mm512_ternarylogic_epi64(z8, z1, z24, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z24, z17, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z17, z10, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 576))); + /* Round 10 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z22, z19, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z11, z8, 0x96); + z26 = z23; + z26 = _mm512_ternarylogic_epi64(z26, z15, z12, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z9, z1, 0x96); + z27 = z16; + z27 = _mm512_ternarylogic_epi64(z27, z13, z5, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z2, z24, 0x96); + z28 = z14; + z28 = _mm512_ternarylogic_epi64(z28, z6, z3, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z20, z17, 0x96); + z29 = z7; + z29 = _mm512_ternarylogic_epi64(z29, z4, z21, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z18, z10, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z22 = _mm512_xor_si512(z22, z30); + z19 = _mm512_xor_si512(z19, z30); + z11 = _mm512_xor_si512(z11, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z23 = _mm512_xor_si512(z23, z30); + z15 = _mm512_xor_si512(z15, z30); + z12 = _mm512_xor_si512(z12, z30); + z9 = _mm512_xor_si512(z9, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z16 = _mm512_xor_si512(z16, z30); + z13 = _mm512_xor_si512(z13, z30); + z5 = _mm512_xor_si512(z5, z30); + z2 = _mm512_xor_si512(z2, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z14 = _mm512_xor_si512(z14, z30); + z6 = _mm512_xor_si512(z6, z30); + z3 = _mm512_xor_si512(z3, z30); + z20 = _mm512_xor_si512(z20, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z7 = _mm512_xor_si512(z7, z30); + z4 = _mm512_xor_si512(z4, z30); + z21 = _mm512_xor_si512(z21, z30); + z18 = _mm512_xor_si512(z18, z30); + z10 = _mm512_xor_si512(z10, z30); + /* Rho - rotate each lane in place */ + z23 = _mm512_rol_epi64(z23, 1); + z16 = _mm512_rol_epi64(z16, 62); + z14 = _mm512_rol_epi64(z14, 28); + z7 = _mm512_rol_epi64(z7, 27); + z22 = _mm512_rol_epi64(z22, 36); + z15 = _mm512_rol_epi64(z15, 44); + z13 = _mm512_rol_epi64(z13, 6); + z6 = _mm512_rol_epi64(z6, 55); + z4 = _mm512_rol_epi64(z4, 20); + z19 = _mm512_rol_epi64(z19, 3); + z12 = _mm512_rol_epi64(z12, 10); + z5 = _mm512_rol_epi64(z5, 43); + z3 = _mm512_rol_epi64(z3, 25); + z21 = _mm512_rol_epi64(z21, 39); + z11 = _mm512_rol_epi64(z11, 41); + z9 = _mm512_rol_epi64(z9, 45); + z2 = _mm512_rol_epi64(z2, 15); + z20 = _mm512_rol_epi64(z20, 21); + z18 = _mm512_rol_epi64(z18, 8); + z8 = _mm512_rol_epi64(z8, 18); + z1 = _mm512_rol_epi64(z1, 2); + z24 = _mm512_rol_epi64(z24, 61); + z17 = _mm512_rol_epi64(z17, 56); + z10 = _mm512_rol_epi64(z10, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z15; + z0 = _mm512_ternarylogic_epi64(z0, z15, z5, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z5, z20, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z20, z10, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z14; + z31 = z4; + z14 = _mm512_ternarylogic_epi64(z14, z4, z19, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z19, z9, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z9, z24, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + z30 = z23; + z31 = z13; + z23 = _mm512_ternarylogic_epi64(z23, z13, z3, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z3, z18, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z18, z8, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z7; + z31 = z22; + z7 = _mm512_ternarylogic_epi64(z7, z22, z12, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z12, z2, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z2, z17, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z16; + z31 = z6; + z16 = _mm512_ternarylogic_epi64(z16, z6, z21, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z21, z11, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z11, z1, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 640))); + /* Round 11 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z14, z23, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z7, z16, 0x96); + z26 = z15; + z26 = _mm512_ternarylogic_epi64(z26, z4, z13, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z22, z6, 0x96); + z27 = z5; + z27 = _mm512_ternarylogic_epi64(z27, z19, z3, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z12, z21, 0x96); + z28 = z20; + z28 = _mm512_ternarylogic_epi64(z28, z9, z18, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z2, z11, 0x96); + z29 = z10; + z29 = _mm512_ternarylogic_epi64(z29, z24, z8, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z17, z1, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z14 = _mm512_xor_si512(z14, z30); + z23 = _mm512_xor_si512(z23, z30); + z7 = _mm512_xor_si512(z7, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z15 = _mm512_xor_si512(z15, z30); + z4 = _mm512_xor_si512(z4, z30); + z13 = _mm512_xor_si512(z13, z30); + z22 = _mm512_xor_si512(z22, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z5 = _mm512_xor_si512(z5, z30); + z19 = _mm512_xor_si512(z19, z30); + z3 = _mm512_xor_si512(z3, z30); + z12 = _mm512_xor_si512(z12, z30); + z21 = _mm512_xor_si512(z21, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z20 = _mm512_xor_si512(z20, z30); + z9 = _mm512_xor_si512(z9, z30); + z18 = _mm512_xor_si512(z18, z30); + z2 = _mm512_xor_si512(z2, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z10 = _mm512_xor_si512(z10, z30); + z24 = _mm512_xor_si512(z24, z30); + z8 = _mm512_xor_si512(z8, z30); + z17 = _mm512_xor_si512(z17, z30); + z1 = _mm512_xor_si512(z1, z30); + /* Rho - rotate each lane in place */ + z15 = _mm512_rol_epi64(z15, 1); + z5 = _mm512_rol_epi64(z5, 62); + z20 = _mm512_rol_epi64(z20, 28); + z10 = _mm512_rol_epi64(z10, 27); + z14 = _mm512_rol_epi64(z14, 36); + z4 = _mm512_rol_epi64(z4, 44); + z19 = _mm512_rol_epi64(z19, 6); + z9 = _mm512_rol_epi64(z9, 55); + z24 = _mm512_rol_epi64(z24, 20); + z23 = _mm512_rol_epi64(z23, 3); + z13 = _mm512_rol_epi64(z13, 10); + z3 = _mm512_rol_epi64(z3, 43); + z18 = _mm512_rol_epi64(z18, 25); + z8 = _mm512_rol_epi64(z8, 39); + z7 = _mm512_rol_epi64(z7, 41); + z22 = _mm512_rol_epi64(z22, 45); + z12 = _mm512_rol_epi64(z12, 15); + z2 = _mm512_rol_epi64(z2, 21); + z17 = _mm512_rol_epi64(z17, 8); + z16 = _mm512_rol_epi64(z16, 18); + z6 = _mm512_rol_epi64(z6, 2); + z21 = _mm512_rol_epi64(z21, 61); + z11 = _mm512_rol_epi64(z11, 56); + z1 = _mm512_rol_epi64(z1, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z4; + z0 = _mm512_ternarylogic_epi64(z0, z4, z3, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z3, z2, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z2, z1, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z20; + z31 = z24; + z20 = _mm512_ternarylogic_epi64(z20, z24, z23, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z23, z22, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z22, z21, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z21, z30, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z30, z31, 0xd2); + z30 = z15; + z31 = z19; + z15 = _mm512_ternarylogic_epi64(z15, z19, z18, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z18, z17, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z17, z16, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z10; + z31 = z14; + z10 = _mm512_ternarylogic_epi64(z10, z14, z13, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z13, z12, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z12, z11, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z5; + z31 = z9; + z5 = _mm512_ternarylogic_epi64(z5, z9, z8, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z8, z7, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z7, z6, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 704))); + /* Round 12 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z20, z15, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z10, z5, 0x96); + z26 = z4; + z26 = _mm512_ternarylogic_epi64(z26, z24, z19, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z14, z9, 0x96); + z27 = z3; + z27 = _mm512_ternarylogic_epi64(z27, z23, z18, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z13, z8, 0x96); + z28 = z2; + z28 = _mm512_ternarylogic_epi64(z28, z22, z17, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z12, z7, 0x96); + z29 = z1; + z29 = _mm512_ternarylogic_epi64(z29, z21, z16, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z11, z6, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z20 = _mm512_xor_si512(z20, z30); + z15 = _mm512_xor_si512(z15, z30); + z10 = _mm512_xor_si512(z10, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z4 = _mm512_xor_si512(z4, z30); + z24 = _mm512_xor_si512(z24, z30); + z19 = _mm512_xor_si512(z19, z30); + z14 = _mm512_xor_si512(z14, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z3 = _mm512_xor_si512(z3, z30); + z23 = _mm512_xor_si512(z23, z30); + z18 = _mm512_xor_si512(z18, z30); + z13 = _mm512_xor_si512(z13, z30); + z8 = _mm512_xor_si512(z8, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z2 = _mm512_xor_si512(z2, z30); + z22 = _mm512_xor_si512(z22, z30); + z17 = _mm512_xor_si512(z17, z30); + z12 = _mm512_xor_si512(z12, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z1 = _mm512_xor_si512(z1, z30); + z21 = _mm512_xor_si512(z21, z30); + z16 = _mm512_xor_si512(z16, z30); + z11 = _mm512_xor_si512(z11, z30); + z6 = _mm512_xor_si512(z6, z30); + /* Rho - rotate each lane in place */ + z4 = _mm512_rol_epi64(z4, 1); + z3 = _mm512_rol_epi64(z3, 62); + z2 = _mm512_rol_epi64(z2, 28); + z1 = _mm512_rol_epi64(z1, 27); + z20 = _mm512_rol_epi64(z20, 36); + z24 = _mm512_rol_epi64(z24, 44); + z23 = _mm512_rol_epi64(z23, 6); + z22 = _mm512_rol_epi64(z22, 55); + z21 = _mm512_rol_epi64(z21, 20); + z15 = _mm512_rol_epi64(z15, 3); + z19 = _mm512_rol_epi64(z19, 10); + z18 = _mm512_rol_epi64(z18, 43); + z17 = _mm512_rol_epi64(z17, 25); + z16 = _mm512_rol_epi64(z16, 39); + z10 = _mm512_rol_epi64(z10, 41); + z14 = _mm512_rol_epi64(z14, 45); + z13 = _mm512_rol_epi64(z13, 15); + z12 = _mm512_rol_epi64(z12, 21); + z11 = _mm512_rol_epi64(z11, 8); + z5 = _mm512_rol_epi64(z5, 18); + z9 = _mm512_rol_epi64(z9, 2); + z8 = _mm512_rol_epi64(z8, 61); + z7 = _mm512_rol_epi64(z7, 56); + z6 = _mm512_rol_epi64(z6, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z24; + z0 = _mm512_ternarylogic_epi64(z0, z24, z18, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z18, z12, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z12, z6, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z2; + z31 = z21; + z2 = _mm512_ternarylogic_epi64(z2, z21, z15, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z15, z14, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z14, z8, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z8, z30, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z30, z31, 0xd2); + z30 = z4; + z31 = z23; + z4 = _mm512_ternarylogic_epi64(z4, z23, z17, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z17, z11, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z11, z5, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z1; + z31 = z20; + z1 = _mm512_ternarylogic_epi64(z1, z20, z19, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z19, z13, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z13, z7, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z3; + z31 = z22; + z3 = _mm512_ternarylogic_epi64(z3, z22, z16, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z16, z10, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z10, z9, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 768))); + /* Round 13 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z2, z4, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z1, z3, 0x96); + z26 = z24; + z26 = _mm512_ternarylogic_epi64(z26, z21, z23, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z20, z22, 0x96); + z27 = z18; + z27 = _mm512_ternarylogic_epi64(z27, z15, z17, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z19, z16, 0x96); + z28 = z12; + z28 = _mm512_ternarylogic_epi64(z28, z14, z11, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z13, z10, 0x96); + z29 = z6; + z29 = _mm512_ternarylogic_epi64(z29, z8, z5, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z7, z9, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z2 = _mm512_xor_si512(z2, z30); + z4 = _mm512_xor_si512(z4, z30); + z1 = _mm512_xor_si512(z1, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z24 = _mm512_xor_si512(z24, z30); + z21 = _mm512_xor_si512(z21, z30); + z23 = _mm512_xor_si512(z23, z30); + z20 = _mm512_xor_si512(z20, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z18 = _mm512_xor_si512(z18, z30); + z15 = _mm512_xor_si512(z15, z30); + z17 = _mm512_xor_si512(z17, z30); + z19 = _mm512_xor_si512(z19, z30); + z16 = _mm512_xor_si512(z16, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z12 = _mm512_xor_si512(z12, z30); + z14 = _mm512_xor_si512(z14, z30); + z11 = _mm512_xor_si512(z11, z30); + z13 = _mm512_xor_si512(z13, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z6 = _mm512_xor_si512(z6, z30); + z8 = _mm512_xor_si512(z8, z30); + z5 = _mm512_xor_si512(z5, z30); + z7 = _mm512_xor_si512(z7, z30); + z9 = _mm512_xor_si512(z9, z30); + /* Rho - rotate each lane in place */ + z24 = _mm512_rol_epi64(z24, 1); + z18 = _mm512_rol_epi64(z18, 62); + z12 = _mm512_rol_epi64(z12, 28); + z6 = _mm512_rol_epi64(z6, 27); + z2 = _mm512_rol_epi64(z2, 36); + z21 = _mm512_rol_epi64(z21, 44); + z15 = _mm512_rol_epi64(z15, 6); + z14 = _mm512_rol_epi64(z14, 55); + z8 = _mm512_rol_epi64(z8, 20); + z4 = _mm512_rol_epi64(z4, 3); + z23 = _mm512_rol_epi64(z23, 10); + z17 = _mm512_rol_epi64(z17, 43); + z11 = _mm512_rol_epi64(z11, 25); + z5 = _mm512_rol_epi64(z5, 39); + z1 = _mm512_rol_epi64(z1, 41); + z20 = _mm512_rol_epi64(z20, 45); + z19 = _mm512_rol_epi64(z19, 15); + z13 = _mm512_rol_epi64(z13, 21); + z7 = _mm512_rol_epi64(z7, 8); + z3 = _mm512_rol_epi64(z3, 18); + z22 = _mm512_rol_epi64(z22, 2); + z16 = _mm512_rol_epi64(z16, 61); + z10 = _mm512_rol_epi64(z10, 56); + z9 = _mm512_rol_epi64(z9, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z21; + z0 = _mm512_ternarylogic_epi64(z0, z21, z17, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z17, z13, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z13, z9, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z12; + z31 = z8; + z12 = _mm512_ternarylogic_epi64(z12, z8, z4, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z4, z20, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z20, z16, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z16, z30, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z30, z31, 0xd2); + z30 = z24; + z31 = z15; + z24 = _mm512_ternarylogic_epi64(z24, z15, z11, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z11, z7, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z7, z3, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z6; + z31 = z2; + z6 = _mm512_ternarylogic_epi64(z6, z2, z23, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z23, z19, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z19, z10, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z18; + z31 = z14; + z18 = _mm512_ternarylogic_epi64(z18, z14, z5, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z5, z1, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z1, z22, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 832))); + /* Round 14 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z12, z24, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z6, z18, 0x96); + z26 = z21; + z26 = _mm512_ternarylogic_epi64(z26, z8, z15, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z2, z14, 0x96); + z27 = z17; + z27 = _mm512_ternarylogic_epi64(z27, z4, z11, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z23, z5, 0x96); + z28 = z13; + z28 = _mm512_ternarylogic_epi64(z28, z20, z7, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z19, z1, 0x96); + z29 = z9; + z29 = _mm512_ternarylogic_epi64(z29, z16, z3, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z10, z22, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z12 = _mm512_xor_si512(z12, z30); + z24 = _mm512_xor_si512(z24, z30); + z6 = _mm512_xor_si512(z6, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z21 = _mm512_xor_si512(z21, z30); + z8 = _mm512_xor_si512(z8, z30); + z15 = _mm512_xor_si512(z15, z30); + z2 = _mm512_xor_si512(z2, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z17 = _mm512_xor_si512(z17, z30); + z4 = _mm512_xor_si512(z4, z30); + z11 = _mm512_xor_si512(z11, z30); + z23 = _mm512_xor_si512(z23, z30); + z5 = _mm512_xor_si512(z5, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z13 = _mm512_xor_si512(z13, z30); + z20 = _mm512_xor_si512(z20, z30); + z7 = _mm512_xor_si512(z7, z30); + z19 = _mm512_xor_si512(z19, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z9 = _mm512_xor_si512(z9, z30); + z16 = _mm512_xor_si512(z16, z30); + z3 = _mm512_xor_si512(z3, z30); + z10 = _mm512_xor_si512(z10, z30); + z22 = _mm512_xor_si512(z22, z30); + /* Rho - rotate each lane in place */ + z21 = _mm512_rol_epi64(z21, 1); + z17 = _mm512_rol_epi64(z17, 62); + z13 = _mm512_rol_epi64(z13, 28); + z9 = _mm512_rol_epi64(z9, 27); + z12 = _mm512_rol_epi64(z12, 36); + z8 = _mm512_rol_epi64(z8, 44); + z4 = _mm512_rol_epi64(z4, 6); + z20 = _mm512_rol_epi64(z20, 55); + z16 = _mm512_rol_epi64(z16, 20); + z24 = _mm512_rol_epi64(z24, 3); + z15 = _mm512_rol_epi64(z15, 10); + z11 = _mm512_rol_epi64(z11, 43); + z7 = _mm512_rol_epi64(z7, 25); + z3 = _mm512_rol_epi64(z3, 39); + z6 = _mm512_rol_epi64(z6, 41); + z2 = _mm512_rol_epi64(z2, 45); + z23 = _mm512_rol_epi64(z23, 15); + z19 = _mm512_rol_epi64(z19, 21); + z10 = _mm512_rol_epi64(z10, 8); + z18 = _mm512_rol_epi64(z18, 18); + z14 = _mm512_rol_epi64(z14, 2); + z5 = _mm512_rol_epi64(z5, 61); + z1 = _mm512_rol_epi64(z1, 56); + z22 = _mm512_rol_epi64(z22, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z8; + z0 = _mm512_ternarylogic_epi64(z0, z8, z11, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z11, z19, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z19, z22, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z13; + z31 = z16; + z13 = _mm512_ternarylogic_epi64(z13, z16, z24, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z24, z2, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z2, z5, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z5, z30, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z30, z31, 0xd2); + z30 = z21; + z31 = z4; + z21 = _mm512_ternarylogic_epi64(z21, z4, z7, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z7, z10, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z10, z18, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z9; + z31 = z12; + z9 = _mm512_ternarylogic_epi64(z9, z12, z15, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z15, z23, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z23, z1, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z17; + z31 = z20; + z17 = _mm512_ternarylogic_epi64(z17, z20, z3, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z3, z6, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z6, z14, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 896))); + /* Round 15 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z13, z21, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z9, z17, 0x96); + z26 = z8; + z26 = _mm512_ternarylogic_epi64(z26, z16, z4, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z12, z20, 0x96); + z27 = z11; + z27 = _mm512_ternarylogic_epi64(z27, z24, z7, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z15, z3, 0x96); + z28 = z19; + z28 = _mm512_ternarylogic_epi64(z28, z2, z10, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z23, z6, 0x96); + z29 = z22; + z29 = _mm512_ternarylogic_epi64(z29, z5, z18, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z1, z14, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z13 = _mm512_xor_si512(z13, z30); + z21 = _mm512_xor_si512(z21, z30); + z9 = _mm512_xor_si512(z9, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z8 = _mm512_xor_si512(z8, z30); + z16 = _mm512_xor_si512(z16, z30); + z4 = _mm512_xor_si512(z4, z30); + z12 = _mm512_xor_si512(z12, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z11 = _mm512_xor_si512(z11, z30); + z24 = _mm512_xor_si512(z24, z30); + z7 = _mm512_xor_si512(z7, z30); + z15 = _mm512_xor_si512(z15, z30); + z3 = _mm512_xor_si512(z3, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z19 = _mm512_xor_si512(z19, z30); + z2 = _mm512_xor_si512(z2, z30); + z10 = _mm512_xor_si512(z10, z30); + z23 = _mm512_xor_si512(z23, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z22 = _mm512_xor_si512(z22, z30); + z5 = _mm512_xor_si512(z5, z30); + z18 = _mm512_xor_si512(z18, z30); + z1 = _mm512_xor_si512(z1, z30); + z14 = _mm512_xor_si512(z14, z30); + /* Rho - rotate each lane in place */ + z8 = _mm512_rol_epi64(z8, 1); + z11 = _mm512_rol_epi64(z11, 62); + z19 = _mm512_rol_epi64(z19, 28); + z22 = _mm512_rol_epi64(z22, 27); + z13 = _mm512_rol_epi64(z13, 36); + z16 = _mm512_rol_epi64(z16, 44); + z24 = _mm512_rol_epi64(z24, 6); + z2 = _mm512_rol_epi64(z2, 55); + z5 = _mm512_rol_epi64(z5, 20); + z21 = _mm512_rol_epi64(z21, 3); + z4 = _mm512_rol_epi64(z4, 10); + z7 = _mm512_rol_epi64(z7, 43); + z10 = _mm512_rol_epi64(z10, 25); + z18 = _mm512_rol_epi64(z18, 39); + z9 = _mm512_rol_epi64(z9, 41); + z12 = _mm512_rol_epi64(z12, 45); + z15 = _mm512_rol_epi64(z15, 15); + z23 = _mm512_rol_epi64(z23, 21); + z1 = _mm512_rol_epi64(z1, 8); + z17 = _mm512_rol_epi64(z17, 18); + z20 = _mm512_rol_epi64(z20, 2); + z3 = _mm512_rol_epi64(z3, 61); + z6 = _mm512_rol_epi64(z6, 56); + z14 = _mm512_rol_epi64(z14, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z16; + z0 = _mm512_ternarylogic_epi64(z0, z16, z7, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z7, z23, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z23, z14, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z19; + z31 = z5; + z19 = _mm512_ternarylogic_epi64(z19, z5, z21, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z21, z12, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z12, z3, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z3, z30, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z30, z31, 0xd2); + z30 = z8; + z31 = z24; + z8 = _mm512_ternarylogic_epi64(z8, z24, z10, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z10, z1, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z1, z17, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z22; + z31 = z13; + z22 = _mm512_ternarylogic_epi64(z22, z13, z4, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z4, z15, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z15, z6, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z11; + z31 = z2; + z11 = _mm512_ternarylogic_epi64(z11, z2, z18, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z18, z9, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z9, z20, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, 960))); + /* Round 16 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z19, z8, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z22, z11, 0x96); + z26 = z16; + z26 = _mm512_ternarylogic_epi64(z26, z5, z24, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z13, z2, 0x96); + z27 = z7; + z27 = _mm512_ternarylogic_epi64(z27, z21, z10, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z4, z18, 0x96); + z28 = z23; + z28 = _mm512_ternarylogic_epi64(z28, z12, z1, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z15, z9, 0x96); + z29 = z14; + z29 = _mm512_ternarylogic_epi64(z29, z3, z17, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z6, z20, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z19 = _mm512_xor_si512(z19, z30); + z8 = _mm512_xor_si512(z8, z30); + z22 = _mm512_xor_si512(z22, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z16 = _mm512_xor_si512(z16, z30); + z5 = _mm512_xor_si512(z5, z30); + z24 = _mm512_xor_si512(z24, z30); + z13 = _mm512_xor_si512(z13, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z7 = _mm512_xor_si512(z7, z30); + z21 = _mm512_xor_si512(z21, z30); + z10 = _mm512_xor_si512(z10, z30); + z4 = _mm512_xor_si512(z4, z30); + z18 = _mm512_xor_si512(z18, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z23 = _mm512_xor_si512(z23, z30); + z12 = _mm512_xor_si512(z12, z30); + z1 = _mm512_xor_si512(z1, z30); + z15 = _mm512_xor_si512(z15, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z14 = _mm512_xor_si512(z14, z30); + z3 = _mm512_xor_si512(z3, z30); + z17 = _mm512_xor_si512(z17, z30); + z6 = _mm512_xor_si512(z6, z30); + z20 = _mm512_xor_si512(z20, z30); + /* Rho - rotate each lane in place */ + z16 = _mm512_rol_epi64(z16, 1); + z7 = _mm512_rol_epi64(z7, 62); + z23 = _mm512_rol_epi64(z23, 28); + z14 = _mm512_rol_epi64(z14, 27); + z19 = _mm512_rol_epi64(z19, 36); + z5 = _mm512_rol_epi64(z5, 44); + z21 = _mm512_rol_epi64(z21, 6); + z12 = _mm512_rol_epi64(z12, 55); + z3 = _mm512_rol_epi64(z3, 20); + z8 = _mm512_rol_epi64(z8, 3); + z24 = _mm512_rol_epi64(z24, 10); + z10 = _mm512_rol_epi64(z10, 43); + z1 = _mm512_rol_epi64(z1, 25); + z17 = _mm512_rol_epi64(z17, 39); + z22 = _mm512_rol_epi64(z22, 41); + z13 = _mm512_rol_epi64(z13, 45); + z4 = _mm512_rol_epi64(z4, 15); + z15 = _mm512_rol_epi64(z15, 21); + z6 = _mm512_rol_epi64(z6, 8); + z11 = _mm512_rol_epi64(z11, 18); + z2 = _mm512_rol_epi64(z2, 2); + z18 = _mm512_rol_epi64(z18, 61); + z9 = _mm512_rol_epi64(z9, 56); + z20 = _mm512_rol_epi64(z20, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z5; + z0 = _mm512_ternarylogic_epi64(z0, z5, z10, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z10, z15, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z15, z20, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z23; + z31 = z3; + z23 = _mm512_ternarylogic_epi64(z23, z3, z8, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z8, z13, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z13, z18, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z18, z30, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z30, z31, 0xd2); + z30 = z16; + z31 = z21; + z16 = _mm512_ternarylogic_epi64(z16, z21, z1, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z1, z6, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z6, z11, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z14; + z31 = z19; + z14 = _mm512_ternarylogic_epi64(z14, z19, z24, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z24, z4, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z4, z9, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z7; + z31 = z12; + z7 = _mm512_ternarylogic_epi64(z7, z12, z17, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z17, z22, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z22, z2, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1024))); + /* Round 17 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z23, z16, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z14, z7, 0x96); + z26 = z5; + z26 = _mm512_ternarylogic_epi64(z26, z3, z21, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z19, z12, 0x96); + z27 = z10; + z27 = _mm512_ternarylogic_epi64(z27, z8, z1, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z24, z17, 0x96); + z28 = z15; + z28 = _mm512_ternarylogic_epi64(z28, z13, z6, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z4, z22, 0x96); + z29 = z20; + z29 = _mm512_ternarylogic_epi64(z29, z18, z11, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z9, z2, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z23 = _mm512_xor_si512(z23, z30); + z16 = _mm512_xor_si512(z16, z30); + z14 = _mm512_xor_si512(z14, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z5 = _mm512_xor_si512(z5, z30); + z3 = _mm512_xor_si512(z3, z30); + z21 = _mm512_xor_si512(z21, z30); + z19 = _mm512_xor_si512(z19, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z10 = _mm512_xor_si512(z10, z30); + z8 = _mm512_xor_si512(z8, z30); + z1 = _mm512_xor_si512(z1, z30); + z24 = _mm512_xor_si512(z24, z30); + z17 = _mm512_xor_si512(z17, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z15 = _mm512_xor_si512(z15, z30); + z13 = _mm512_xor_si512(z13, z30); + z6 = _mm512_xor_si512(z6, z30); + z4 = _mm512_xor_si512(z4, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z20 = _mm512_xor_si512(z20, z30); + z18 = _mm512_xor_si512(z18, z30); + z11 = _mm512_xor_si512(z11, z30); + z9 = _mm512_xor_si512(z9, z30); + z2 = _mm512_xor_si512(z2, z30); + /* Rho - rotate each lane in place */ + z5 = _mm512_rol_epi64(z5, 1); + z10 = _mm512_rol_epi64(z10, 62); + z15 = _mm512_rol_epi64(z15, 28); + z20 = _mm512_rol_epi64(z20, 27); + z23 = _mm512_rol_epi64(z23, 36); + z3 = _mm512_rol_epi64(z3, 44); + z8 = _mm512_rol_epi64(z8, 6); + z13 = _mm512_rol_epi64(z13, 55); + z18 = _mm512_rol_epi64(z18, 20); + z16 = _mm512_rol_epi64(z16, 3); + z21 = _mm512_rol_epi64(z21, 10); + z1 = _mm512_rol_epi64(z1, 43); + z6 = _mm512_rol_epi64(z6, 25); + z11 = _mm512_rol_epi64(z11, 39); + z14 = _mm512_rol_epi64(z14, 41); + z19 = _mm512_rol_epi64(z19, 45); + z24 = _mm512_rol_epi64(z24, 15); + z4 = _mm512_rol_epi64(z4, 21); + z9 = _mm512_rol_epi64(z9, 8); + z7 = _mm512_rol_epi64(z7, 18); + z12 = _mm512_rol_epi64(z12, 2); + z17 = _mm512_rol_epi64(z17, 61); + z22 = _mm512_rol_epi64(z22, 56); + z2 = _mm512_rol_epi64(z2, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z3; + z0 = _mm512_ternarylogic_epi64(z0, z3, z1, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z1, z4, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z4, z2, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z15; + z31 = z18; + z15 = _mm512_ternarylogic_epi64(z15, z18, z16, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z16, z19, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z19, z17, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z17, z30, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z30, z31, 0xd2); + z30 = z5; + z31 = z8; + z5 = _mm512_ternarylogic_epi64(z5, z8, z6, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z6, z9, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z9, z7, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z20; + z31 = z23; + z20 = _mm512_ternarylogic_epi64(z20, z23, z21, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z21, z24, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z24, z22, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z10; + z31 = z13; + z10 = _mm512_ternarylogic_epi64(z10, z13, z11, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z11, z14, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z14, z12, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1088))); + /* Round 18 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z15, z5, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z20, z10, 0x96); + z26 = z3; + z26 = _mm512_ternarylogic_epi64(z26, z18, z8, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z23, z13, 0x96); + z27 = z1; + z27 = _mm512_ternarylogic_epi64(z27, z16, z6, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z21, z11, 0x96); + z28 = z4; + z28 = _mm512_ternarylogic_epi64(z28, z19, z9, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z24, z14, 0x96); + z29 = z2; + z29 = _mm512_ternarylogic_epi64(z29, z17, z7, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z22, z12, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z15 = _mm512_xor_si512(z15, z30); + z5 = _mm512_xor_si512(z5, z30); + z20 = _mm512_xor_si512(z20, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z3 = _mm512_xor_si512(z3, z30); + z18 = _mm512_xor_si512(z18, z30); + z8 = _mm512_xor_si512(z8, z30); + z23 = _mm512_xor_si512(z23, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z1 = _mm512_xor_si512(z1, z30); + z16 = _mm512_xor_si512(z16, z30); + z6 = _mm512_xor_si512(z6, z30); + z21 = _mm512_xor_si512(z21, z30); + z11 = _mm512_xor_si512(z11, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z4 = _mm512_xor_si512(z4, z30); + z19 = _mm512_xor_si512(z19, z30); + z9 = _mm512_xor_si512(z9, z30); + z24 = _mm512_xor_si512(z24, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z2 = _mm512_xor_si512(z2, z30); + z17 = _mm512_xor_si512(z17, z30); + z7 = _mm512_xor_si512(z7, z30); + z22 = _mm512_xor_si512(z22, z30); + z12 = _mm512_xor_si512(z12, z30); + /* Rho - rotate each lane in place */ + z3 = _mm512_rol_epi64(z3, 1); + z1 = _mm512_rol_epi64(z1, 62); + z4 = _mm512_rol_epi64(z4, 28); + z2 = _mm512_rol_epi64(z2, 27); + z15 = _mm512_rol_epi64(z15, 36); + z18 = _mm512_rol_epi64(z18, 44); + z16 = _mm512_rol_epi64(z16, 6); + z19 = _mm512_rol_epi64(z19, 55); + z17 = _mm512_rol_epi64(z17, 20); + z5 = _mm512_rol_epi64(z5, 3); + z8 = _mm512_rol_epi64(z8, 10); + z6 = _mm512_rol_epi64(z6, 43); + z9 = _mm512_rol_epi64(z9, 25); + z7 = _mm512_rol_epi64(z7, 39); + z20 = _mm512_rol_epi64(z20, 41); + z23 = _mm512_rol_epi64(z23, 45); + z21 = _mm512_rol_epi64(z21, 15); + z24 = _mm512_rol_epi64(z24, 21); + z22 = _mm512_rol_epi64(z22, 8); + z10 = _mm512_rol_epi64(z10, 18); + z13 = _mm512_rol_epi64(z13, 2); + z11 = _mm512_rol_epi64(z11, 61); + z14 = _mm512_rol_epi64(z14, 56); + z12 = _mm512_rol_epi64(z12, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z18; + z0 = _mm512_ternarylogic_epi64(z0, z18, z6, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z6, z24, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z24, z12, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z4; + z31 = z17; + z4 = _mm512_ternarylogic_epi64(z4, z17, z5, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z5, z23, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z23, z11, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z11, z30, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z30, z31, 0xd2); + z30 = z3; + z31 = z16; + z3 = _mm512_ternarylogic_epi64(z3, z16, z9, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z9, z22, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z22, z10, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z2; + z31 = z15; + z2 = _mm512_ternarylogic_epi64(z2, z15, z8, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z8, z21, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z21, z14, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z1; + z31 = z19; + z1 = _mm512_ternarylogic_epi64(z1, z19, z7, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z7, z20, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z20, z13, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1152))); + /* Round 19 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z4, z3, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z2, z1, 0x96); + z26 = z18; + z26 = _mm512_ternarylogic_epi64(z26, z17, z16, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z15, z19, 0x96); + z27 = z6; + z27 = _mm512_ternarylogic_epi64(z27, z5, z9, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z8, z7, 0x96); + z28 = z24; + z28 = _mm512_ternarylogic_epi64(z28, z23, z22, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z21, z20, 0x96); + z29 = z12; + z29 = _mm512_ternarylogic_epi64(z29, z11, z10, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z14, z13, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z4 = _mm512_xor_si512(z4, z30); + z3 = _mm512_xor_si512(z3, z30); + z2 = _mm512_xor_si512(z2, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z18 = _mm512_xor_si512(z18, z30); + z17 = _mm512_xor_si512(z17, z30); + z16 = _mm512_xor_si512(z16, z30); + z15 = _mm512_xor_si512(z15, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z6 = _mm512_xor_si512(z6, z30); + z5 = _mm512_xor_si512(z5, z30); + z9 = _mm512_xor_si512(z9, z30); + z8 = _mm512_xor_si512(z8, z30); + z7 = _mm512_xor_si512(z7, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z24 = _mm512_xor_si512(z24, z30); + z23 = _mm512_xor_si512(z23, z30); + z22 = _mm512_xor_si512(z22, z30); + z21 = _mm512_xor_si512(z21, z30); + z20 = _mm512_xor_si512(z20, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z12 = _mm512_xor_si512(z12, z30); + z11 = _mm512_xor_si512(z11, z30); + z10 = _mm512_xor_si512(z10, z30); + z14 = _mm512_xor_si512(z14, z30); + z13 = _mm512_xor_si512(z13, z30); + /* Rho - rotate each lane in place */ + z18 = _mm512_rol_epi64(z18, 1); + z6 = _mm512_rol_epi64(z6, 62); + z24 = _mm512_rol_epi64(z24, 28); + z12 = _mm512_rol_epi64(z12, 27); + z4 = _mm512_rol_epi64(z4, 36); + z17 = _mm512_rol_epi64(z17, 44); + z5 = _mm512_rol_epi64(z5, 6); + z23 = _mm512_rol_epi64(z23, 55); + z11 = _mm512_rol_epi64(z11, 20); + z3 = _mm512_rol_epi64(z3, 3); + z16 = _mm512_rol_epi64(z16, 10); + z9 = _mm512_rol_epi64(z9, 43); + z22 = _mm512_rol_epi64(z22, 25); + z10 = _mm512_rol_epi64(z10, 39); + z2 = _mm512_rol_epi64(z2, 41); + z15 = _mm512_rol_epi64(z15, 45); + z8 = _mm512_rol_epi64(z8, 15); + z21 = _mm512_rol_epi64(z21, 21); + z14 = _mm512_rol_epi64(z14, 8); + z1 = _mm512_rol_epi64(z1, 18); + z19 = _mm512_rol_epi64(z19, 2); + z7 = _mm512_rol_epi64(z7, 61); + z20 = _mm512_rol_epi64(z20, 56); + z13 = _mm512_rol_epi64(z13, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z17; + z0 = _mm512_ternarylogic_epi64(z0, z17, z9, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z9, z21, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z21, z13, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z24; + z31 = z11; + z24 = _mm512_ternarylogic_epi64(z24, z11, z3, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z3, z15, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z15, z7, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z7, z30, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z30, z31, 0xd2); + z30 = z18; + z31 = z5; + z18 = _mm512_ternarylogic_epi64(z18, z5, z22, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z22, z14, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z14, z1, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z12; + z31 = z4; + z12 = _mm512_ternarylogic_epi64(z12, z4, z16, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z16, z8, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z8, z20, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z20, z30, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z30, z31, 0xd2); + z30 = z6; + z31 = z23; + z6 = _mm512_ternarylogic_epi64(z6, z23, z10, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z10, z2, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z2, z19, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1216))); + /* Round 20 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z24, z18, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z12, z6, 0x96); + z26 = z17; + z26 = _mm512_ternarylogic_epi64(z26, z11, z5, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z4, z23, 0x96); + z27 = z9; + z27 = _mm512_ternarylogic_epi64(z27, z3, z22, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z16, z10, 0x96); + z28 = z21; + z28 = _mm512_ternarylogic_epi64(z28, z15, z14, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z8, z2, 0x96); + z29 = z13; + z29 = _mm512_ternarylogic_epi64(z29, z7, z1, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z20, z19, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z24 = _mm512_xor_si512(z24, z30); + z18 = _mm512_xor_si512(z18, z30); + z12 = _mm512_xor_si512(z12, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z17 = _mm512_xor_si512(z17, z30); + z11 = _mm512_xor_si512(z11, z30); + z5 = _mm512_xor_si512(z5, z30); + z4 = _mm512_xor_si512(z4, z30); + z23 = _mm512_xor_si512(z23, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z9 = _mm512_xor_si512(z9, z30); + z3 = _mm512_xor_si512(z3, z30); + z22 = _mm512_xor_si512(z22, z30); + z16 = _mm512_xor_si512(z16, z30); + z10 = _mm512_xor_si512(z10, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z21 = _mm512_xor_si512(z21, z30); + z15 = _mm512_xor_si512(z15, z30); + z14 = _mm512_xor_si512(z14, z30); + z8 = _mm512_xor_si512(z8, z30); + z2 = _mm512_xor_si512(z2, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z13 = _mm512_xor_si512(z13, z30); + z7 = _mm512_xor_si512(z7, z30); + z1 = _mm512_xor_si512(z1, z30); + z20 = _mm512_xor_si512(z20, z30); + z19 = _mm512_xor_si512(z19, z30); + /* Rho - rotate each lane in place */ + z17 = _mm512_rol_epi64(z17, 1); + z9 = _mm512_rol_epi64(z9, 62); + z21 = _mm512_rol_epi64(z21, 28); + z13 = _mm512_rol_epi64(z13, 27); + z24 = _mm512_rol_epi64(z24, 36); + z11 = _mm512_rol_epi64(z11, 44); + z3 = _mm512_rol_epi64(z3, 6); + z15 = _mm512_rol_epi64(z15, 55); + z7 = _mm512_rol_epi64(z7, 20); + z18 = _mm512_rol_epi64(z18, 3); + z5 = _mm512_rol_epi64(z5, 10); + z22 = _mm512_rol_epi64(z22, 43); + z14 = _mm512_rol_epi64(z14, 25); + z1 = _mm512_rol_epi64(z1, 39); + z12 = _mm512_rol_epi64(z12, 41); + z4 = _mm512_rol_epi64(z4, 45); + z16 = _mm512_rol_epi64(z16, 15); + z8 = _mm512_rol_epi64(z8, 21); + z20 = _mm512_rol_epi64(z20, 8); + z6 = _mm512_rol_epi64(z6, 18); + z23 = _mm512_rol_epi64(z23, 2); + z10 = _mm512_rol_epi64(z10, 61); + z2 = _mm512_rol_epi64(z2, 56); + z19 = _mm512_rol_epi64(z19, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z11; + z0 = _mm512_ternarylogic_epi64(z0, z11, z22, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z22, z8, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z8, z19, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z21; + z31 = z7; + z21 = _mm512_ternarylogic_epi64(z21, z7, z18, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z18, z4, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z4, z10, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z10, z30, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z30, z31, 0xd2); + z30 = z17; + z31 = z3; + z17 = _mm512_ternarylogic_epi64(z17, z3, z14, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z14, z20, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z20, z6, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z13; + z31 = z24; + z13 = _mm512_ternarylogic_epi64(z13, z24, z5, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z5, z16, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z16, z2, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z2, z30, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z30, z31, 0xd2); + z30 = z9; + z31 = z15; + z9 = _mm512_ternarylogic_epi64(z9, z15, z1, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z1, z12, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z12, z23, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1280))); + /* Round 21 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z21, z17, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z13, z9, 0x96); + z26 = z11; + z26 = _mm512_ternarylogic_epi64(z26, z7, z3, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z24, z15, 0x96); + z27 = z22; + z27 = _mm512_ternarylogic_epi64(z27, z18, z14, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z5, z1, 0x96); + z28 = z8; + z28 = _mm512_ternarylogic_epi64(z28, z4, z20, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z16, z12, 0x96); + z29 = z19; + z29 = _mm512_ternarylogic_epi64(z29, z10, z6, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z2, z23, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z21 = _mm512_xor_si512(z21, z30); + z17 = _mm512_xor_si512(z17, z30); + z13 = _mm512_xor_si512(z13, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z11 = _mm512_xor_si512(z11, z30); + z7 = _mm512_xor_si512(z7, z30); + z3 = _mm512_xor_si512(z3, z30); + z24 = _mm512_xor_si512(z24, z30); + z15 = _mm512_xor_si512(z15, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z22 = _mm512_xor_si512(z22, z30); + z18 = _mm512_xor_si512(z18, z30); + z14 = _mm512_xor_si512(z14, z30); + z5 = _mm512_xor_si512(z5, z30); + z1 = _mm512_xor_si512(z1, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z8 = _mm512_xor_si512(z8, z30); + z4 = _mm512_xor_si512(z4, z30); + z20 = _mm512_xor_si512(z20, z30); + z16 = _mm512_xor_si512(z16, z30); + z12 = _mm512_xor_si512(z12, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z19 = _mm512_xor_si512(z19, z30); + z10 = _mm512_xor_si512(z10, z30); + z6 = _mm512_xor_si512(z6, z30); + z2 = _mm512_xor_si512(z2, z30); + z23 = _mm512_xor_si512(z23, z30); + /* Rho - rotate each lane in place */ + z11 = _mm512_rol_epi64(z11, 1); + z22 = _mm512_rol_epi64(z22, 62); + z8 = _mm512_rol_epi64(z8, 28); + z19 = _mm512_rol_epi64(z19, 27); + z21 = _mm512_rol_epi64(z21, 36); + z7 = _mm512_rol_epi64(z7, 44); + z18 = _mm512_rol_epi64(z18, 6); + z4 = _mm512_rol_epi64(z4, 55); + z10 = _mm512_rol_epi64(z10, 20); + z17 = _mm512_rol_epi64(z17, 3); + z3 = _mm512_rol_epi64(z3, 10); + z14 = _mm512_rol_epi64(z14, 43); + z20 = _mm512_rol_epi64(z20, 25); + z6 = _mm512_rol_epi64(z6, 39); + z13 = _mm512_rol_epi64(z13, 41); + z24 = _mm512_rol_epi64(z24, 45); + z5 = _mm512_rol_epi64(z5, 15); + z16 = _mm512_rol_epi64(z16, 21); + z2 = _mm512_rol_epi64(z2, 8); + z9 = _mm512_rol_epi64(z9, 18); + z15 = _mm512_rol_epi64(z15, 2); + z1 = _mm512_rol_epi64(z1, 61); + z12 = _mm512_rol_epi64(z12, 56); + z23 = _mm512_rol_epi64(z23, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z7; + z0 = _mm512_ternarylogic_epi64(z0, z7, z14, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z14, z16, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z16, z23, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z23, z30, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z30, z31, 0xd2); + z30 = z8; + z31 = z10; + z8 = _mm512_ternarylogic_epi64(z8, z10, z17, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z17, z24, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z24, z1, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z1, z30, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z30, z31, 0xd2); + z30 = z11; + z31 = z18; + z11 = _mm512_ternarylogic_epi64(z11, z18, z20, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z20, z2, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z2, z9, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z19; + z31 = z21; + z19 = _mm512_ternarylogic_epi64(z19, z21, z3, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z3, z5, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z5, z12, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z12, z30, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z30, z31, 0xd2); + z30 = z22; + z31 = z4; + z22 = _mm512_ternarylogic_epi64(z22, z4, z6, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z6, z13, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z13, z15, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1344))); + /* Round 22 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z8, z11, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z19, z22, 0x96); + z26 = z7; + z26 = _mm512_ternarylogic_epi64(z26, z10, z18, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z21, z4, 0x96); + z27 = z14; + z27 = _mm512_ternarylogic_epi64(z27, z17, z20, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z3, z6, 0x96); + z28 = z16; + z28 = _mm512_ternarylogic_epi64(z28, z24, z2, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z5, z13, 0x96); + z29 = z23; + z29 = _mm512_ternarylogic_epi64(z29, z1, z9, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z12, z15, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z8 = _mm512_xor_si512(z8, z30); + z11 = _mm512_xor_si512(z11, z30); + z19 = _mm512_xor_si512(z19, z30); + z22 = _mm512_xor_si512(z22, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z7 = _mm512_xor_si512(z7, z30); + z10 = _mm512_xor_si512(z10, z30); + z18 = _mm512_xor_si512(z18, z30); + z21 = _mm512_xor_si512(z21, z30); + z4 = _mm512_xor_si512(z4, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z14 = _mm512_xor_si512(z14, z30); + z17 = _mm512_xor_si512(z17, z30); + z20 = _mm512_xor_si512(z20, z30); + z3 = _mm512_xor_si512(z3, z30); + z6 = _mm512_xor_si512(z6, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z16 = _mm512_xor_si512(z16, z30); + z24 = _mm512_xor_si512(z24, z30); + z2 = _mm512_xor_si512(z2, z30); + z5 = _mm512_xor_si512(z5, z30); + z13 = _mm512_xor_si512(z13, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z23 = _mm512_xor_si512(z23, z30); + z1 = _mm512_xor_si512(z1, z30); + z9 = _mm512_xor_si512(z9, z30); + z12 = _mm512_xor_si512(z12, z30); + z15 = _mm512_xor_si512(z15, z30); + /* Rho - rotate each lane in place */ + z7 = _mm512_rol_epi64(z7, 1); + z14 = _mm512_rol_epi64(z14, 62); + z16 = _mm512_rol_epi64(z16, 28); + z23 = _mm512_rol_epi64(z23, 27); + z8 = _mm512_rol_epi64(z8, 36); + z10 = _mm512_rol_epi64(z10, 44); + z17 = _mm512_rol_epi64(z17, 6); + z24 = _mm512_rol_epi64(z24, 55); + z1 = _mm512_rol_epi64(z1, 20); + z11 = _mm512_rol_epi64(z11, 3); + z18 = _mm512_rol_epi64(z18, 10); + z20 = _mm512_rol_epi64(z20, 43); + z2 = _mm512_rol_epi64(z2, 25); + z9 = _mm512_rol_epi64(z9, 39); + z19 = _mm512_rol_epi64(z19, 41); + z21 = _mm512_rol_epi64(z21, 45); + z3 = _mm512_rol_epi64(z3, 15); + z5 = _mm512_rol_epi64(z5, 21); + z12 = _mm512_rol_epi64(z12, 8); + z22 = _mm512_rol_epi64(z22, 18); + z4 = _mm512_rol_epi64(z4, 2); + z6 = _mm512_rol_epi64(z6, 61); + z13 = _mm512_rol_epi64(z13, 56); + z15 = _mm512_rol_epi64(z15, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z10; + z0 = _mm512_ternarylogic_epi64(z0, z10, z20, 0xd2); + z10 = _mm512_ternarylogic_epi64(z10, z20, z5, 0xd2); + z20 = _mm512_ternarylogic_epi64(z20, z5, z15, 0xd2); + z5 = _mm512_ternarylogic_epi64(z5, z15, z30, 0xd2); + z15 = _mm512_ternarylogic_epi64(z15, z30, z31, 0xd2); + z30 = z16; + z31 = z1; + z16 = _mm512_ternarylogic_epi64(z16, z1, z11, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z11, z21, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z21, z6, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z6, z30, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z30, z31, 0xd2); + z30 = z7; + z31 = z17; + z7 = _mm512_ternarylogic_epi64(z7, z17, z2, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z2, z12, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z12, z22, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z22, z30, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z30, z31, 0xd2); + z30 = z23; + z31 = z8; + z23 = _mm512_ternarylogic_epi64(z23, z8, z18, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z18, z3, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z3, z13, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z13, z30, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z30, z31, 0xd2); + z30 = z14; + z31 = z24; + z14 = _mm512_ternarylogic_epi64(z14, z24, z9, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z9, z19, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z19, z4, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1408))); + /* Round 23 */ + /* Theta - column parity */ + z25 = z0; + z25 = _mm512_ternarylogic_epi64(z25, z16, z7, 0x96); + z25 = _mm512_ternarylogic_epi64(z25, z23, z14, 0x96); + z26 = z10; + z26 = _mm512_ternarylogic_epi64(z26, z1, z17, 0x96); + z26 = _mm512_ternarylogic_epi64(z26, z8, z24, 0x96); + z27 = z20; + z27 = _mm512_ternarylogic_epi64(z27, z11, z2, 0x96); + z27 = _mm512_ternarylogic_epi64(z27, z18, z9, 0x96); + z28 = z5; + z28 = _mm512_ternarylogic_epi64(z28, z21, z12, 0x96); + z28 = _mm512_ternarylogic_epi64(z28, z3, z19, 0x96); + z29 = z15; + z29 = _mm512_ternarylogic_epi64(z29, z6, z22, 0x96); + z29 = _mm512_ternarylogic_epi64(z29, z13, z4, 0x96); + /* Theta - D and apply in place */ + z30 = _mm512_rol_epi64(z26, 1); + z30 = _mm512_xor_si512(z30, z29); + z0 = _mm512_xor_si512(z0, z30); + z16 = _mm512_xor_si512(z16, z30); + z7 = _mm512_xor_si512(z7, z30); + z23 = _mm512_xor_si512(z23, z30); + z14 = _mm512_xor_si512(z14, z30); + z30 = _mm512_rol_epi64(z27, 1); + z30 = _mm512_xor_si512(z30, z25); + z10 = _mm512_xor_si512(z10, z30); + z1 = _mm512_xor_si512(z1, z30); + z17 = _mm512_xor_si512(z17, z30); + z8 = _mm512_xor_si512(z8, z30); + z24 = _mm512_xor_si512(z24, z30); + z30 = _mm512_rol_epi64(z28, 1); + z30 = _mm512_xor_si512(z30, z26); + z20 = _mm512_xor_si512(z20, z30); + z11 = _mm512_xor_si512(z11, z30); + z2 = _mm512_xor_si512(z2, z30); + z18 = _mm512_xor_si512(z18, z30); + z9 = _mm512_xor_si512(z9, z30); + z30 = _mm512_rol_epi64(z29, 1); + z30 = _mm512_xor_si512(z30, z27); + z5 = _mm512_xor_si512(z5, z30); + z21 = _mm512_xor_si512(z21, z30); + z12 = _mm512_xor_si512(z12, z30); + z3 = _mm512_xor_si512(z3, z30); + z19 = _mm512_xor_si512(z19, z30); + z30 = _mm512_rol_epi64(z25, 1); + z30 = _mm512_xor_si512(z30, z28); + z15 = _mm512_xor_si512(z15, z30); + z6 = _mm512_xor_si512(z6, z30); + z22 = _mm512_xor_si512(z22, z30); + z13 = _mm512_xor_si512(z13, z30); + z4 = _mm512_xor_si512(z4, z30); + /* Rho - rotate each lane in place */ + z10 = _mm512_rol_epi64(z10, 1); + z20 = _mm512_rol_epi64(z20, 62); + z5 = _mm512_rol_epi64(z5, 28); + z15 = _mm512_rol_epi64(z15, 27); + z16 = _mm512_rol_epi64(z16, 36); + z1 = _mm512_rol_epi64(z1, 44); + z11 = _mm512_rol_epi64(z11, 6); + z21 = _mm512_rol_epi64(z21, 55); + z6 = _mm512_rol_epi64(z6, 20); + z7 = _mm512_rol_epi64(z7, 3); + z17 = _mm512_rol_epi64(z17, 10); + z2 = _mm512_rol_epi64(z2, 43); + z12 = _mm512_rol_epi64(z12, 25); + z22 = _mm512_rol_epi64(z22, 39); + z23 = _mm512_rol_epi64(z23, 41); + z8 = _mm512_rol_epi64(z8, 45); + z18 = _mm512_rol_epi64(z18, 15); + z3 = _mm512_rol_epi64(z3, 21); + z13 = _mm512_rol_epi64(z13, 8); + z14 = _mm512_rol_epi64(z14, 18); + z24 = _mm512_rol_epi64(z24, 2); + z9 = _mm512_rol_epi64(z9, 61); + z19 = _mm512_rol_epi64(z19, 56); + z4 = _mm512_rol_epi64(z4, 14); + /* Pi - relabel logical -> physical */ + /* Chi - per row, in place, fused with vpternlogq (imm 0xd2) */ + z30 = z0; + z31 = z1; + z0 = _mm512_ternarylogic_epi64(z0, z1, z2, 0xd2); + z1 = _mm512_ternarylogic_epi64(z1, z2, z3, 0xd2); + z2 = _mm512_ternarylogic_epi64(z2, z3, z4, 0xd2); + z3 = _mm512_ternarylogic_epi64(z3, z4, z30, 0xd2); + z4 = _mm512_ternarylogic_epi64(z4, z30, z31, 0xd2); + z30 = z5; + z31 = z6; + z5 = _mm512_ternarylogic_epi64(z5, z6, z7, 0xd2); + z6 = _mm512_ternarylogic_epi64(z6, z7, z8, 0xd2); + z7 = _mm512_ternarylogic_epi64(z7, z8, z9, 0xd2); + z8 = _mm512_ternarylogic_epi64(z8, z9, z30, 0xd2); + z9 = _mm512_ternarylogic_epi64(z9, z30, z31, 0xd2); + z30 = z10; + z31 = z11; + z10 = _mm512_ternarylogic_epi64(z10, z11, z12, 0xd2); + z11 = _mm512_ternarylogic_epi64(z11, z12, z13, 0xd2); + z12 = _mm512_ternarylogic_epi64(z12, z13, z14, 0xd2); + z13 = _mm512_ternarylogic_epi64(z13, z14, z30, 0xd2); + z14 = _mm512_ternarylogic_epi64(z14, z30, z31, 0xd2); + z30 = z15; + z31 = z16; + z15 = _mm512_ternarylogic_epi64(z15, z16, z17, 0xd2); + z16 = _mm512_ternarylogic_epi64(z16, z17, z18, 0xd2); + z17 = _mm512_ternarylogic_epi64(z17, z18, z19, 0xd2); + z18 = _mm512_ternarylogic_epi64(z18, z19, z30, 0xd2); + z19 = _mm512_ternarylogic_epi64(z19, z30, z31, 0xd2); + z30 = z20; + z31 = z21; + z20 = _mm512_ternarylogic_epi64(z20, z21, z22, 0xd2); + z21 = _mm512_ternarylogic_epi64(z21, z22, z23, 0xd2); + z22 = _mm512_ternarylogic_epi64(z22, z23, z24, 0xd2); + z23 = _mm512_ternarylogic_epi64(z23, z24, z30, 0xd2); + z24 = _mm512_ternarylogic_epi64(z24, z30, z31, 0xd2); + /* Iota */ + z0 = _mm512_xor_si512(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + 1472))); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 1024), z16); + _mm512_storeu_si512((void*)WC_PW(rdi, 1088), z17); + _mm512_storeu_si512((void*)WC_PW(rdi, 1152), z18); + _mm512_storeu_si512((void*)WC_PW(rdi, 1216), z19); + _mm512_storeu_si512((void*)WC_PW(rdi, 1280), z20); + _mm512_storeu_si512((void*)WC_PW(rdi, 1344), z21); + _mm512_storeu_si512((void*)WC_PW(rdi, 1408), z22); + _mm512_storeu_si512((void*)WC_PW(rdi, 1472), z23); + _mm512_storeu_si512((void*)WC_PW(rdi, 1536), z24); +} + +#endif /* WOLFSSL_HAVE_MLDSA */ +#endif /* HAVE_INTEL_AVX512 */ +#endif /* HAVE_INTEL_AVX2 */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/sha512_asm.asm b/wolfcrypt/src/sha512_asm.asm index 0b95a23dae0..b7cdf467fee 100644 --- a/wolfcrypt/src/sha512_asm.asm +++ b/wolfcrypt/src/sha512_asm.asm @@ -6443,7 +6443,9 @@ Transform_Sha512_AVX2_Len PROC vmovups YMMWORD PTR [rdi+96], ymm1 vmovups YMMWORD PTR [rdi+128], ymm2 vmovups YMMWORD PTR [rdi+160], ymm3 + sub rsp, 32 call Transform_Sha512_AVX2 + add rsp, 32 add QWORD PTR [rdi+224], 128 sub ebp, 128 jz L_sha512_len_avx2_done @@ -9241,9 +9243,9 @@ Transform_Sha512_AVX2_RORX_Len PROC vmovups YMMWORD PTR [rdi+96], ymm1 vmovups YMMWORD PTR [rdi+128], ymm2 vmovups YMMWORD PTR [rdi+160], ymm3 - sub rsp, 8 + sub rsp, 40 call Transform_Sha512_AVX2_RORX - add rsp, 8 + add rsp, 40 pop rsi add QWORD PTR [rdi+224], 128 sub esi, 128 diff --git a/wolfcrypt/src/sha512_x86_64_intrin.c b/wolfcrypt/src/sha512_x86_64_intrin.c new file mode 100644 index 00000000000..7ca1d9e3e20 --- /dev/null +++ b/wolfcrypt/src/sha512_x86_64_intrin.c @@ -0,0 +1,10912 @@ +/* sha512_x86_64_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_SHA512_X86_64_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef HAVE_INTEL_AVX1 +extern WOLFSSL_LOCAL int Transform_Sha512_AVX1(wc_Sha512* sha512); +extern WOLFSSL_LOCAL int Transform_Sha512_AVX1_Len(wc_Sha512* sha512, + word32 len); +extern WOLFSSL_LOCAL int Transform_Sha512_AVX1_RORX(wc_Sha512* sha512); +extern WOLFSSL_LOCAL int Transform_Sha512_AVX1_RORX_Len(wc_Sha512* sha512, + word32 len); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL int Transform_Sha512_AVX2(wc_Sha512* sha512); +extern WOLFSSL_LOCAL int Transform_Sha512_AVX2_Len(wc_Sha512* sha512, + word32 len); +extern WOLFSSL_LOCAL int Transform_Sha512_AVX2_RORX(wc_Sha512* sha512); +extern WOLFSSL_LOCAL int Transform_Sha512_AVX2_RORX_Len(wc_Sha512* sha512, + word32 len); + +#endif +#ifdef HAVE_INTEL_AVX1 +XALIGNED(32) static const word64 L_avx1_sha512_k[] WC_X64I_UNUSED = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, +}; + +XALIGNED(16) static const word64 L_avx1_sha512_flip_mask[] WC_X64I_UNUSED = { + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, +}; + +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL int Transform_Sha512_AVX1(wc_Sha512* sha512) +{ + word64 rdi, rsp, rax, r8, r9, r10, r11, r12, r13, r14, r15, rsi, rbx, + rdx = 0, rcx = 0; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), + x13 = _mm_setzero_si128(), x14; + XALIGNED(32) WC_X64I_SLOT stk[20]; + + rdi = (word64)(size_t)sha512; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 136); + rax = (word64)(rdi + 64); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha512_flip_mask, 0)); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 16)); + x0 = _mm_shuffle_epi8(x0, x14); + x1 = _mm_shuffle_epi8(x1, x14); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 48)); + x2 = _mm_shuffle_epi8(x2, x14); + x3 = _mm_shuffle_epi8(x3, x14); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 80)); + x4 = _mm_shuffle_epi8(x4, x14); + x5 = _mm_shuffle_epi8(x5, x14); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 112)); + x6 = _mm_shuffle_epi8(x6, x14); + x7 = _mm_shuffle_epi8(x7, x14); + WC_S32(rsp, 128) = (word32)(4); + rsi = (word64)((word64)(size_t)L_avx1_sha512_k); + rbx = (word64)(r9); + rax = (word64)(r12); + rbx = (word64)(rbx ^ r10); + /* Start of 16 rounds */ +L_transform_sha512_avx1_start: + x8 = _mm_add_epi64(x0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x9 = _mm_add_epi64(x1, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x9); + x8 = _mm_add_epi64(x2, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32))); + x9 = _mm_add_epi64(x3, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x9); + x8 = _mm_add_epi64(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64))); + x9 = _mm_add_epi64(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x9); + x8 = _mm_add_epi64(x6, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96))); + x9 = _mm_add_epi64(x7, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x9); + rsi = (word64)(rsi + 0x80); + /* msg_sched: 0-1 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x1, x0, 8); + x13 = _mm_alignr_epi8(x5, x4, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rcx = (word64)(rcx ^ r14); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + x8 = _mm_xor_si128(x8, x11); + x0 = _mm_add_epi64(x13, x0); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x0 = _mm_add_epi64(x8, x0); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rcx = (word64)(rcx ^ r13); + x8 = _mm_srli_epi64(x7, 19); + x9 = _mm_slli_epi64(x7, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + x10 = _mm_srli_epi64(x7, 61); + x11 = _mm_slli_epi64(x7, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x7, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + x0 = _mm_add_epi64(x8, x0); + /* msg_sched done: 0-1 */ + /* msg_sched: 2-3 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x2, x1, 8); + x13 = _mm_alignr_epi8(x6, x5, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rcx = (word64)(rcx ^ r12); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + x8 = _mm_xor_si128(x8, x11); + x1 = _mm_add_epi64(x13, x1); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x1 = _mm_add_epi64(x8, x1); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ r11); + x8 = _mm_srli_epi64(x0, 19); + x9 = _mm_slli_epi64(x0, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + x10 = _mm_srli_epi64(x0, 61); + x11 = _mm_slli_epi64(x0, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x0, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + x1 = _mm_add_epi64(x8, x1); + /* msg_sched done: 2-3 */ + /* msg_sched: 4-5 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x3, x2, 8); + x13 = _mm_alignr_epi8(x7, x6, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rcx = (word64)(rcx ^ r10); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + x8 = _mm_xor_si128(x8, x11); + x2 = _mm_add_epi64(x13, x2); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x2 = _mm_add_epi64(x8, x2); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rcx = (word64)(rcx ^ r9); + x8 = _mm_srli_epi64(x1, 19); + x9 = _mm_slli_epi64(x1, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + x10 = _mm_srli_epi64(x1, 61); + x11 = _mm_slli_epi64(x1, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x1, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + x2 = _mm_add_epi64(x8, x2); + /* msg_sched done: 4-5 */ + /* msg_sched: 6-7 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x4, x3, 8); + x13 = _mm_alignr_epi8(x0, x7, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rcx = (word64)(rcx ^ r8); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + x8 = _mm_xor_si128(x8, x11); + x3 = _mm_add_epi64(x13, x3); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x3 = _mm_add_epi64(x8, x3); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rcx = (word64)(rcx ^ r15); + x8 = _mm_srli_epi64(x2, 19); + x9 = _mm_slli_epi64(x2, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + x10 = _mm_srli_epi64(x2, 61); + x11 = _mm_slli_epi64(x2, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x2, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + x3 = _mm_add_epi64(x8, x3); + /* msg_sched done: 6-7 */ + /* msg_sched: 8-9 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x5, x4, 8); + x13 = _mm_alignr_epi8(x1, x0, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rcx = (word64)(rcx ^ r14); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + x8 = _mm_xor_si128(x8, x11); + x4 = _mm_add_epi64(x13, x4); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x4 = _mm_add_epi64(x8, x4); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rcx = (word64)(rcx ^ r13); + x8 = _mm_srli_epi64(x3, 19); + x9 = _mm_slli_epi64(x3, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + x10 = _mm_srli_epi64(x3, 61); + x11 = _mm_slli_epi64(x3, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x3, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + x4 = _mm_add_epi64(x8, x4); + /* msg_sched done: 8-9 */ + /* msg_sched: 10-11 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x6, x5, 8); + x13 = _mm_alignr_epi8(x2, x1, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rcx = (word64)(rcx ^ r12); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + x8 = _mm_xor_si128(x8, x11); + x5 = _mm_add_epi64(x13, x5); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x5 = _mm_add_epi64(x8, x5); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rcx = (word64)(rcx ^ r11); + x8 = _mm_srli_epi64(x4, 19); + x9 = _mm_slli_epi64(x4, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + x10 = _mm_srli_epi64(x4, 61); + x11 = _mm_slli_epi64(x4, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x4, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + x5 = _mm_add_epi64(x8, x5); + /* msg_sched done: 10-11 */ + /* msg_sched: 12-13 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x7, x6, 8); + x13 = _mm_alignr_epi8(x3, x2, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rcx = (word64)(rcx ^ r10); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + x8 = _mm_xor_si128(x8, x11); + x6 = _mm_add_epi64(x13, x6); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x6 = _mm_add_epi64(x8, x6); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rcx = (word64)(rcx ^ r9); + x8 = _mm_srli_epi64(x5, 19); + x9 = _mm_slli_epi64(x5, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + x10 = _mm_srli_epi64(x5, 61); + x11 = _mm_slli_epi64(x5, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x5, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + x6 = _mm_add_epi64(x8, x6); + /* msg_sched done: 12-13 */ + /* msg_sched: 14-15 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x0, x7, 8); + x13 = _mm_alignr_epi8(x4, x3, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rcx = (word64)(rcx ^ r8); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + x8 = _mm_xor_si128(x8, x11); + x7 = _mm_add_epi64(x13, x7); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x7 = _mm_add_epi64(x8, x7); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rcx = (word64)(rcx ^ r15); + x8 = _mm_srli_epi64(x6, 19); + x9 = _mm_slli_epi64(x6, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + x10 = _mm_srli_epi64(x6, 61); + x11 = _mm_slli_epi64(x6, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x6, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + x7 = _mm_add_epi64(x8, x7); + /* msg_sched done: 14-15 */ + WC_S32(rsp, 128) = (word32)(WC_L32(rsp, 128) - 1); + if (((word32)WC_S32(rsp, 128)) != (0)) { + goto L_transform_sha512_avx1_start; + } + x8 = _mm_add_epi64(x0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x9 = _mm_add_epi64(x1, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x9); + x8 = _mm_add_epi64(x2, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32))); + x9 = _mm_add_epi64(x3, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x9); + x8 = _mm_add_epi64(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64))); + x9 = _mm_add_epi64(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x9); + x8 = _mm_add_epi64(x6, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96))); + x9 = _mm_add_epi64(x7, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x9); + /* rnd_all_2: 0-1 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + /* rnd_all_2: 2-3 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + /* rnd_all_2: 4-5 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + /* rnd_all_2: 6-7 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + /* rnd_all_2: 8-9 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + /* rnd_all_2: 10-11 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + /* rnd_all_2: 12-13 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + /* rnd_all_2: 14-15 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) + r8); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) + r9); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) + r10); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) + r11); + WC_S64(rdi, 32) = (word64)(WC_L64(rdi, 32) + r12); + WC_S64(rdi, 40) = (word64)(WC_L64(rdi, 40) + r13); + WC_S64(rdi, 48) = (word64)(WC_L64(rdi, 48) + r14); + WC_S64(rdi, 56) = (word64)(WC_L64(rdi, 56) + r15); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL int Transform_Sha512_AVX1_Len(wc_Sha512* sha512, word32 len) +{ + word64 rdi, rbp, rsp, rsi, rdx, r8, r9, r10, r11, r12, r13, r14, r15, + rbx = 0, rax = 0, rcx = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), x14; + XALIGNED(32) WC_X64I_SLOT stk[20]; + word32 zf1; + + rdi = (word64)(size_t)sha512; + rbp = (word64)(word32)len; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 144); + rsi = (word64)(WC_L64(rdi, 224)); + rdx = (word64)((word64)(size_t)L_avx1_sha512_k); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_sha512_flip_mask, 0)); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + /* Start of loop processing a block */ +L_sha512_len_avx1_begin: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_shuffle_epi8(x0, x14); + x1 = _mm_shuffle_epi8(x1, x14); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x2 = _mm_shuffle_epi8(x2, x14); + x3 = _mm_shuffle_epi8(x3, x14); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x4 = _mm_shuffle_epi8(x4, x14); + x5 = _mm_shuffle_epi8(x5, x14); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x6 = _mm_shuffle_epi8(x6, x14); + x7 = _mm_shuffle_epi8(x7, x14); + WC_S32(rsp, 128) = (word32)(4); + rbx = (word64)(r9); + rax = (word64)(r12); + rbx = (word64)(rbx ^ r10); + x8 = _mm_add_epi64(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x9 = _mm_add_epi64(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x9); + x8 = _mm_add_epi64(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x9 = _mm_add_epi64(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x9); + x8 = _mm_add_epi64(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x9 = _mm_add_epi64(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x9); + x8 = _mm_add_epi64(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x9 = _mm_add_epi64(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x9); + /* Start of 16 rounds */ +L_sha512_len_avx1_start: + rdx = (word64)(rdx + 0x80); + WC_S64(rsp, 136) = (word64)(rdx); + /* msg_sched: 0-1 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x1, x0, 8); + x13 = _mm_alignr_epi8(x5, x4, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rcx = (word64)(rcx ^ r14); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + x8 = _mm_xor_si128(x8, x11); + x0 = _mm_add_epi64(x13, x0); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x0 = _mm_add_epi64(x8, x0); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rcx = (word64)(rcx ^ r13); + x8 = _mm_srli_epi64(x7, 19); + x9 = _mm_slli_epi64(x7, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + x10 = _mm_srli_epi64(x7, 61); + x11 = _mm_slli_epi64(x7, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x7, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + x0 = _mm_add_epi64(x8, x0); + /* msg_sched done: 0-1 */ + /* msg_sched: 2-3 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x2, x1, 8); + x13 = _mm_alignr_epi8(x6, x5, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rcx = (word64)(rcx ^ r12); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + x8 = _mm_xor_si128(x8, x11); + x1 = _mm_add_epi64(x13, x1); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x1 = _mm_add_epi64(x8, x1); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ r11); + x8 = _mm_srli_epi64(x0, 19); + x9 = _mm_slli_epi64(x0, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + x10 = _mm_srli_epi64(x0, 61); + x11 = _mm_slli_epi64(x0, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x0, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + x1 = _mm_add_epi64(x8, x1); + /* msg_sched done: 2-3 */ + /* msg_sched: 4-5 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x3, x2, 8); + x13 = _mm_alignr_epi8(x7, x6, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rcx = (word64)(rcx ^ r10); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + x8 = _mm_xor_si128(x8, x11); + x2 = _mm_add_epi64(x13, x2); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x2 = _mm_add_epi64(x8, x2); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rcx = (word64)(rcx ^ r9); + x8 = _mm_srli_epi64(x1, 19); + x9 = _mm_slli_epi64(x1, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + x10 = _mm_srli_epi64(x1, 61); + x11 = _mm_slli_epi64(x1, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x1, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + x2 = _mm_add_epi64(x8, x2); + /* msg_sched done: 4-5 */ + /* msg_sched: 6-7 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x4, x3, 8); + x13 = _mm_alignr_epi8(x0, x7, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rcx = (word64)(rcx ^ r8); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + x8 = _mm_xor_si128(x8, x11); + x3 = _mm_add_epi64(x13, x3); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x3 = _mm_add_epi64(x8, x3); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rcx = (word64)(rcx ^ r15); + x8 = _mm_srli_epi64(x2, 19); + x9 = _mm_slli_epi64(x2, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + x10 = _mm_srli_epi64(x2, 61); + x11 = _mm_slli_epi64(x2, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x2, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + x3 = _mm_add_epi64(x8, x3); + /* msg_sched done: 6-7 */ + /* msg_sched: 8-9 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x5, x4, 8); + x13 = _mm_alignr_epi8(x1, x0, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rcx = (word64)(rcx ^ r14); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + x8 = _mm_xor_si128(x8, x11); + x4 = _mm_add_epi64(x13, x4); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x4 = _mm_add_epi64(x8, x4); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rcx = (word64)(rcx ^ r13); + x8 = _mm_srli_epi64(x3, 19); + x9 = _mm_slli_epi64(x3, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + x10 = _mm_srli_epi64(x3, 61); + x11 = _mm_slli_epi64(x3, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x3, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + x4 = _mm_add_epi64(x8, x4); + /* msg_sched done: 8-9 */ + /* msg_sched: 10-11 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x6, x5, 8); + x13 = _mm_alignr_epi8(x2, x1, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rcx = (word64)(rcx ^ r12); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + x8 = _mm_xor_si128(x8, x11); + x5 = _mm_add_epi64(x13, x5); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x5 = _mm_add_epi64(x8, x5); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rcx = (word64)(rcx ^ r11); + x8 = _mm_srli_epi64(x4, 19); + x9 = _mm_slli_epi64(x4, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + x10 = _mm_srli_epi64(x4, 61); + x11 = _mm_slli_epi64(x4, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x4, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + x5 = _mm_add_epi64(x8, x5); + /* msg_sched done: 10-11 */ + /* msg_sched: 12-13 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x7, x6, 8); + x13 = _mm_alignr_epi8(x3, x2, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rcx = (word64)(rcx ^ r10); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + x8 = _mm_xor_si128(x8, x11); + x6 = _mm_add_epi64(x13, x6); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x6 = _mm_add_epi64(x8, x6); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rcx = (word64)(rcx ^ r9); + x8 = _mm_srli_epi64(x5, 19); + x9 = _mm_slli_epi64(x5, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + x10 = _mm_srli_epi64(x5, 61); + x11 = _mm_slli_epi64(x5, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x5, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + x6 = _mm_add_epi64(x8, x6); + /* msg_sched done: 12-13 */ + /* msg_sched: 14-15 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x12 = _mm_alignr_epi8(x0, x7, 8); + x13 = _mm_alignr_epi8(x4, x3, 8); + /* rnd_0: 1 - 1 */ + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rcx = (word64)(rcx ^ r8); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 3 */ + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 4 - 5 */ + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 6 - 7 */ + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 8 - 9 */ + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + x8 = _mm_xor_si128(x8, x11); + x7 = _mm_add_epi64(x13, x7); + /* rnd_0: 10 - 11 */ + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + /* rnd_1: 0 - 0 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + x7 = _mm_add_epi64(x8, x7); + /* rnd_1: 1 - 1 */ + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rcx = (word64)(rcx ^ r15); + x8 = _mm_srli_epi64(x6, 19); + x9 = _mm_slli_epi64(x6, 45); + /* rnd_1: 2 - 3 */ + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + x10 = _mm_srli_epi64(x6, 61); + x11 = _mm_slli_epi64(x6, 3); + /* rnd_1: 4 - 6 */ + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 7 - 8 */ + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x6, 6); + /* rnd_1: 9 - 10 */ + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 11 - 11 */ + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + x7 = _mm_add_epi64(x8, x7); + /* msg_sched done: 14-15 */ + rdx = (word64)(WC_L64(rsp, 136)); + x8 = _mm_add_epi64(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x9 = _mm_add_epi64(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x9); + x8 = _mm_add_epi64(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x9 = _mm_add_epi64(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x9); + x8 = _mm_add_epi64(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x9 = _mm_add_epi64(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x9); + x8 = _mm_add_epi64(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x9 = _mm_add_epi64(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x9); + WC_S32(rsp, 128) = (word32)(WC_L32(rsp, 128) - 1); + if (((word32)WC_S32(rsp, 128)) != (0)) { + goto L_sha512_len_avx1_start; + } + /* rnd_all_2: 0-1 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + /* rnd_all_2: 2-3 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + /* rnd_all_2: 4-5 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + /* rnd_all_2: 6-7 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + /* rnd_all_2: 8-9 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + /* rnd_all_2: 10-11 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + /* rnd_all_2: 12-13 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + /* rnd_all_2: 14-15 */ + /* rnd_0: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + /* rnd_1: 0 - 11 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + r8 = (word64)(r8 + WC_L64(rdi, 0)); + r9 = (word64)(r9 + WC_L64(rdi, 8)); + r10 = (word64)(r10 + WC_L64(rdi, 16)); + r11 = (word64)(r11 + WC_L64(rdi, 24)); + r12 = (word64)(r12 + WC_L64(rdi, 32)); + r13 = (word64)(r13 + WC_L64(rdi, 40)); + r14 = (word64)(r14 + WC_L64(rdi, 48)); + r15 = (word64)(r15 + WC_L64(rdi, 56)); + rdx = (word64)((word64)(size_t)L_avx1_sha512_k); + rsi = (word64)(rsi + 0x80); + rbp = (word32)((word32)rbp - 0x80); + zf1 = (word32)rbp; + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rdi, 32) = (word64)(r12); + WC_S64(rdi, 40) = (word64)(r13); + WC_S64(rdi, 48) = (word64)(r14); + WC_S64(rdi, 56) = (word64)(r15); + if ((zf1) != (0)) { + goto L_sha512_len_avx1_begin; + } + rax = (word64)(0); + return (int)(word32)rax; +} + +XALIGNED(32) static const word64 L_avx1_rorx_sha512_k[] WC_X64I_UNUSED = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, +}; + +XALIGNED(16) static const word64 L_avx1_rorx_sha512_flip_mask[] + WC_X64I_UNUSED = { + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, +}; + +WC_X64I_TARGET("avx,bmi2") +WOLFSSL_LOCAL int Transform_Sha512_AVX1_RORX(wc_Sha512* sha512) +{ + word64 rdi, rsp, rax, r8, r9, r10, r11, r12, r13, r14, r15, rsi, rbx, rdx, + rcx = 0; + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), + x13 = _mm_setzero_si128(), x14; + XALIGNED(32) WC_X64I_SLOT stk[20]; + + rdi = (word64)(size_t)sha512; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 136); + rax = (word64)(rdi + 64); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_rorx_sha512_flip_mask, + 0)); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 16)); + x0 = _mm_shuffle_epi8(x0, x14); + x1 = _mm_shuffle_epi8(x1, x14); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 48)); + x2 = _mm_shuffle_epi8(x2, x14); + x3 = _mm_shuffle_epi8(x3, x14); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 80)); + x4 = _mm_shuffle_epi8(x4, x14); + x5 = _mm_shuffle_epi8(x5, x14); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rax, 112)); + x6 = _mm_shuffle_epi8(x6, x14); + x7 = _mm_shuffle_epi8(x7, x14); + WC_S32(rsp, 128) = (word32)(4); + rsi = (word64)((word64)(size_t)L_avx1_rorx_sha512_k); + rbx = (word64)(r9); + rdx = (word64)(0); + rbx = (word64)(rbx ^ r10); + x8 = _mm_add_epi64(x0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x9 = _mm_add_epi64(x1, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x9); + x8 = _mm_add_epi64(x2, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32))); + x9 = _mm_add_epi64(x3, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x9); + x8 = _mm_add_epi64(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64))); + x9 = _mm_add_epi64(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x9); + x8 = _mm_add_epi64(x6, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96))); + x9 = _mm_add_epi64(x7, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x9); + /* Start of 16 rounds */ +L_transform_sha512_avx1_rorx_start: + rsi = (word64)(rsi + 0x80); + /* msg_sched: 0-1 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + x12 = _mm_alignr_epi8(x1, x0, 8); + x13 = _mm_alignr_epi8(x5, x4, 8); + /* rnd_0: 1 - 1 */ + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x0 = _mm_add_epi64(x13, x0); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + x0 = _mm_add_epi64(x8, x0); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + x8 = _mm_srli_epi64(x7, 19); + x9 = _mm_slli_epi64(x7, 45); + /* rnd_1: 1 - 1 */ + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x7, 61); + x11 = _mm_slli_epi64(x7, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x7, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + x0 = _mm_add_epi64(x8, x0); + /* msg_sched done: 0-1 */ + /* msg_sched: 2-3 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + x12 = _mm_alignr_epi8(x2, x1, 8); + x13 = _mm_alignr_epi8(x6, x5, 8); + /* rnd_0: 1 - 1 */ + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x1 = _mm_add_epi64(x13, x1); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + x1 = _mm_add_epi64(x8, x1); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + x8 = _mm_srli_epi64(x0, 19); + x9 = _mm_slli_epi64(x0, 45); + /* rnd_1: 1 - 1 */ + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x0, 61); + x11 = _mm_slli_epi64(x0, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x0, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + x1 = _mm_add_epi64(x8, x1); + /* msg_sched done: 2-3 */ + /* msg_sched: 4-5 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + x12 = _mm_alignr_epi8(x3, x2, 8); + x13 = _mm_alignr_epi8(x7, x6, 8); + /* rnd_0: 1 - 1 */ + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x2 = _mm_add_epi64(x13, x2); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + x2 = _mm_add_epi64(x8, x2); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + x8 = _mm_srli_epi64(x1, 19); + x9 = _mm_slli_epi64(x1, 45); + /* rnd_1: 1 - 1 */ + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x1, 61); + x11 = _mm_slli_epi64(x1, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x1, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + x2 = _mm_add_epi64(x8, x2); + /* msg_sched done: 4-5 */ + /* msg_sched: 6-7 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + x12 = _mm_alignr_epi8(x4, x3, 8); + x13 = _mm_alignr_epi8(x0, x7, 8); + /* rnd_0: 1 - 1 */ + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x3 = _mm_add_epi64(x13, x3); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + x3 = _mm_add_epi64(x8, x3); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + x8 = _mm_srli_epi64(x2, 19); + x9 = _mm_slli_epi64(x2, 45); + /* rnd_1: 1 - 1 */ + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x2, 61); + x11 = _mm_slli_epi64(x2, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x2, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + x3 = _mm_add_epi64(x8, x3); + /* msg_sched done: 6-7 */ + /* msg_sched: 8-9 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + x12 = _mm_alignr_epi8(x5, x4, 8); + x13 = _mm_alignr_epi8(x1, x0, 8); + /* rnd_0: 1 - 1 */ + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x4 = _mm_add_epi64(x13, x4); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + x4 = _mm_add_epi64(x8, x4); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + x8 = _mm_srli_epi64(x3, 19); + x9 = _mm_slli_epi64(x3, 45); + /* rnd_1: 1 - 1 */ + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x3, 61); + x11 = _mm_slli_epi64(x3, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x3, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + x4 = _mm_add_epi64(x8, x4); + /* msg_sched done: 8-9 */ + /* msg_sched: 10-11 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + x12 = _mm_alignr_epi8(x6, x5, 8); + x13 = _mm_alignr_epi8(x2, x1, 8); + /* rnd_0: 1 - 1 */ + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x5 = _mm_add_epi64(x13, x5); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + x5 = _mm_add_epi64(x8, x5); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + x8 = _mm_srli_epi64(x4, 19); + x9 = _mm_slli_epi64(x4, 45); + /* rnd_1: 1 - 1 */ + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x4, 61); + x11 = _mm_slli_epi64(x4, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x4, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + x5 = _mm_add_epi64(x8, x5); + /* msg_sched done: 10-11 */ + /* msg_sched: 12-13 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + x12 = _mm_alignr_epi8(x7, x6, 8); + x13 = _mm_alignr_epi8(x3, x2, 8); + /* rnd_0: 1 - 1 */ + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x6 = _mm_add_epi64(x13, x6); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + x6 = _mm_add_epi64(x8, x6); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + x8 = _mm_srli_epi64(x5, 19); + x9 = _mm_slli_epi64(x5, 45); + /* rnd_1: 1 - 1 */ + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x5, 61); + x11 = _mm_slli_epi64(x5, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x5, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + x6 = _mm_add_epi64(x8, x6); + /* msg_sched done: 12-13 */ + /* msg_sched: 14-15 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + x12 = _mm_alignr_epi8(x0, x7, 8); + x13 = _mm_alignr_epi8(x4, x3, 8); + /* rnd_0: 1 - 1 */ + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x7 = _mm_add_epi64(x13, x7); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + x7 = _mm_add_epi64(x8, x7); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + x8 = _mm_srli_epi64(x6, 19); + x9 = _mm_slli_epi64(x6, 45); + /* rnd_1: 1 - 1 */ + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x6, 61); + x11 = _mm_slli_epi64(x6, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x6, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + x7 = _mm_add_epi64(x8, x7); + /* msg_sched done: 14-15 */ + x8 = _mm_add_epi64(x0, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + x9 = _mm_add_epi64(x1, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x9); + x8 = _mm_add_epi64(x2, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32))); + x9 = _mm_add_epi64(x3, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x9); + x8 = _mm_add_epi64(x4, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64))); + x9 = _mm_add_epi64(x5, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x9); + x8 = _mm_add_epi64(x6, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96))); + x9 = _mm_add_epi64(x7, _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x9); + WC_S32(rsp, 128) = (word32)(WC_L32(rsp, 128) - 1); + if (((word32)WC_S32(rsp, 128)) != (0)) { + goto L_transform_sha512_avx1_rorx_start; + } + /* rnd_all_2: 0-1 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + /* rnd_all_2: 2-3 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_2: 4-5 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + /* rnd_all_2: 6-7 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + /* rnd_all_2: 8-9 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + /* rnd_all_2: 10-11 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_2: 12-13 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + /* rnd_all_2: 14-15 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + r8 = (word64)(r8 + rdx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) + r8); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) + r9); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) + r10); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) + r11); + WC_S64(rdi, 32) = (word64)(WC_L64(rdi, 32) + r12); + WC_S64(rdi, 40) = (word64)(WC_L64(rdi, 40) + r13); + WC_S64(rdi, 48) = (word64)(WC_L64(rdi, 48) + r14); + WC_S64(rdi, 56) = (word64)(WC_L64(rdi, 56) + r15); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx,bmi2") +WOLFSSL_LOCAL int Transform_Sha512_AVX1_RORX_Len(wc_Sha512* sha512, word32 len) +{ + word64 rdi, rbp, rsp, rsi, rcx, r8, r9, r10, r11, r12, r13, r14, r15, + rbx = 0, rdx = 0, rax = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(), x14; + XALIGNED(32) WC_X64I_SLOT stk[20]; + word32 zf1; + + rdi = (word64)(size_t)sha512; + rbp = (word64)(word32)len; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 144); + rsi = (word64)(WC_L64(rdi, 224)); + rcx = (word64)((word64)(size_t)L_avx1_rorx_sha512_k); + x14 = _mm_loadu_si128((const __m128i*)WC_PR(L_avx1_rorx_sha512_flip_mask, + 0)); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + /* Start of loop processing a block */ +L_sha512_len_avx1_rorx_begin: + x0 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x1 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x0 = _mm_shuffle_epi8(x0, x14); + x1 = _mm_shuffle_epi8(x1, x14); + x2 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x3 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x2 = _mm_shuffle_epi8(x2, x14); + x3 = _mm_shuffle_epi8(x3, x14); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x4 = _mm_shuffle_epi8(x4, x14); + x5 = _mm_shuffle_epi8(x5, x14); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x6 = _mm_shuffle_epi8(x6, x14); + x7 = _mm_shuffle_epi8(x7, x14); + WC_S32(rsp, 128) = (word32)(4); + rbx = (word64)(r9); + rdx = (word64)(0); + rbx = (word64)(rbx ^ r10); + x8 = _mm_add_epi64(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x9 = _mm_add_epi64(x1, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x9); + x8 = _mm_add_epi64(x2, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32))); + x9 = _mm_add_epi64(x3, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x9); + x8 = _mm_add_epi64(x4, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64))); + x9 = _mm_add_epi64(x5, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x9); + x8 = _mm_add_epi64(x6, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96))); + x9 = _mm_add_epi64(x7, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x9); + /* Start of 16 rounds */ +L_sha512_len_avx1_rorx_start: + rcx = (word64)(rcx + 0x80); + WC_S64(rsp, 136) = (word64)(rcx); + /* msg_sched: 0-1 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + x12 = _mm_alignr_epi8(x1, x0, 8); + x13 = _mm_alignr_epi8(x5, x4, 8); + /* rnd_0: 1 - 1 */ + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x0 = _mm_add_epi64(x13, x0); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + x0 = _mm_add_epi64(x8, x0); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + x8 = _mm_srli_epi64(x7, 19); + x9 = _mm_slli_epi64(x7, 45); + /* rnd_1: 1 - 1 */ + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x7, 61); + x11 = _mm_slli_epi64(x7, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x7, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + x0 = _mm_add_epi64(x8, x0); + /* msg_sched done: 0-1 */ + /* msg_sched: 2-3 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + x12 = _mm_alignr_epi8(x2, x1, 8); + x13 = _mm_alignr_epi8(x6, x5, 8); + /* rnd_0: 1 - 1 */ + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x1 = _mm_add_epi64(x13, x1); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + x1 = _mm_add_epi64(x8, x1); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + x8 = _mm_srli_epi64(x0, 19); + x9 = _mm_slli_epi64(x0, 45); + /* rnd_1: 1 - 1 */ + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x0, 61); + x11 = _mm_slli_epi64(x0, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x0, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + x1 = _mm_add_epi64(x8, x1); + /* msg_sched done: 2-3 */ + /* msg_sched: 4-5 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + x12 = _mm_alignr_epi8(x3, x2, 8); + x13 = _mm_alignr_epi8(x7, x6, 8); + /* rnd_0: 1 - 1 */ + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x2 = _mm_add_epi64(x13, x2); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + x2 = _mm_add_epi64(x8, x2); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + x8 = _mm_srli_epi64(x1, 19); + x9 = _mm_slli_epi64(x1, 45); + /* rnd_1: 1 - 1 */ + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x1, 61); + x11 = _mm_slli_epi64(x1, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x1, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + x2 = _mm_add_epi64(x8, x2); + /* msg_sched done: 4-5 */ + /* msg_sched: 6-7 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + x12 = _mm_alignr_epi8(x4, x3, 8); + x13 = _mm_alignr_epi8(x0, x7, 8); + /* rnd_0: 1 - 1 */ + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x3 = _mm_add_epi64(x13, x3); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + x3 = _mm_add_epi64(x8, x3); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + x8 = _mm_srli_epi64(x2, 19); + x9 = _mm_slli_epi64(x2, 45); + /* rnd_1: 1 - 1 */ + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x2, 61); + x11 = _mm_slli_epi64(x2, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x2, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + x3 = _mm_add_epi64(x8, x3); + /* msg_sched done: 6-7 */ + /* msg_sched: 8-9 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + x12 = _mm_alignr_epi8(x5, x4, 8); + x13 = _mm_alignr_epi8(x1, x0, 8); + /* rnd_0: 1 - 1 */ + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x4 = _mm_add_epi64(x13, x4); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + x4 = _mm_add_epi64(x8, x4); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + x8 = _mm_srli_epi64(x3, 19); + x9 = _mm_slli_epi64(x3, 45); + /* rnd_1: 1 - 1 */ + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x3, 61); + x11 = _mm_slli_epi64(x3, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x3, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + x4 = _mm_add_epi64(x8, x4); + /* msg_sched done: 8-9 */ + /* msg_sched: 10-11 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + x12 = _mm_alignr_epi8(x6, x5, 8); + x13 = _mm_alignr_epi8(x2, x1, 8); + /* rnd_0: 1 - 1 */ + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x5 = _mm_add_epi64(x13, x5); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + x5 = _mm_add_epi64(x8, x5); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + x8 = _mm_srli_epi64(x4, 19); + x9 = _mm_slli_epi64(x4, 45); + /* rnd_1: 1 - 1 */ + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x4, 61); + x11 = _mm_slli_epi64(x4, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x4, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + x5 = _mm_add_epi64(x8, x5); + /* msg_sched done: 10-11 */ + /* msg_sched: 12-13 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + x12 = _mm_alignr_epi8(x7, x6, 8); + x13 = _mm_alignr_epi8(x3, x2, 8); + /* rnd_0: 1 - 1 */ + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x6 = _mm_add_epi64(x13, x6); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + x6 = _mm_add_epi64(x8, x6); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + x8 = _mm_srli_epi64(x5, 19); + x9 = _mm_slli_epi64(x5, 45); + /* rnd_1: 1 - 1 */ + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x5, 61); + x11 = _mm_slli_epi64(x5, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x5, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + x6 = _mm_add_epi64(x8, x6); + /* msg_sched done: 12-13 */ + /* msg_sched: 14-15 */ + /* rnd_0: 0 - 0 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + x12 = _mm_alignr_epi8(x0, x7, 8); + x13 = _mm_alignr_epi8(x4, x3, 8); + /* rnd_0: 1 - 1 */ + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + x8 = _mm_srli_epi64(x12, 1); + x9 = _mm_slli_epi64(x12, 63); + /* rnd_0: 2 - 2 */ + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + x10 = _mm_srli_epi64(x12, 8); + x11 = _mm_slli_epi64(x12, 56); + /* rnd_0: 3 - 3 */ + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_0: 4 - 4 */ + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + x11 = _mm_srli_epi64(x12, 7); + x8 = _mm_xor_si128(x8, x10); + /* rnd_0: 5 - 5 */ + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + x8 = _mm_xor_si128(x8, x11); + x7 = _mm_add_epi64(x13, x7); + /* rnd_0: 6 - 7 */ + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + x7 = _mm_add_epi64(x8, x7); + /* rnd_1: 0 - 0 */ + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + x8 = _mm_srli_epi64(x6, 19); + x9 = _mm_slli_epi64(x6, 45); + /* rnd_1: 1 - 1 */ + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + x10 = _mm_srli_epi64(x6, 61); + x11 = _mm_slli_epi64(x6, 3); + /* rnd_1: 2 - 2 */ + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + x8 = _mm_or_si128(x8, x9); + x10 = _mm_or_si128(x10, x11); + /* rnd_1: 3 - 4 */ + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + x8 = _mm_xor_si128(x8, x10); + x11 = _mm_srli_epi64(x6, 6); + /* rnd_1: 5 - 6 */ + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + x8 = _mm_xor_si128(x8, x11); + /* rnd_1: 7 - 7 */ + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + x7 = _mm_add_epi64(x8, x7); + /* msg_sched done: 14-15 */ + rcx = (word64)(WC_L64(rsp, 136)); + x8 = _mm_add_epi64(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x9 = _mm_add_epi64(x1, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x9); + x8 = _mm_add_epi64(x2, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32))); + x9 = _mm_add_epi64(x3, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x9); + x8 = _mm_add_epi64(x4, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64))); + x9 = _mm_add_epi64(x5, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x9); + x8 = _mm_add_epi64(x6, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96))); + x9 = _mm_add_epi64(x7, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x9); + WC_S32(rsp, 128) = (word32)(WC_L32(rsp, 128) - 1); + if (((word32)WC_S32(rsp, 128)) != (0)) { + goto L_sha512_len_avx1_rorx_start; + } + x8 = _mm_add_epi64(x0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + x9 = _mm_add_epi64(x1, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 16))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 0), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 16), x9); + x8 = _mm_add_epi64(x2, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 32))); + x9 = _mm_add_epi64(x3, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 48))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 32), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 48), x9); + x8 = _mm_add_epi64(x4, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 64))); + x9 = _mm_add_epi64(x5, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 80))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 64), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 80), x9); + x8 = _mm_add_epi64(x6, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 96))); + x9 = _mm_add_epi64(x7, _mm_loadu_si128((const __m128i*)WC_PR(rcx, 112))); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 96), x8); + _mm_storeu_si128((__m128i*)WC_PW(rsp, 112), x9); + /* rnd_all_2: 0-1 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + /* rnd_all_2: 2-3 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_2: 4-5 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + /* rnd_all_2: 6-7 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + /* rnd_all_2: 8-9 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + /* rnd_all_2: 10-11 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_2: 12-13 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + /* rnd_all_2: 14-15 */ + /* rnd_0: 0 - 7 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + /* rnd_1: 0 - 7 */ + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + r8 = (word64)(r8 + rdx); + r8 = (word64)(r8 + WC_L64(rdi, 0)); + r9 = (word64)(r9 + WC_L64(rdi, 8)); + r10 = (word64)(r10 + WC_L64(rdi, 16)); + r11 = (word64)(r11 + WC_L64(rdi, 24)); + r12 = (word64)(r12 + WC_L64(rdi, 32)); + r13 = (word64)(r13 + WC_L64(rdi, 40)); + r14 = (word64)(r14 + WC_L64(rdi, 48)); + r15 = (word64)(r15 + WC_L64(rdi, 56)); + rcx = (word64)((word64)(size_t)L_avx1_rorx_sha512_k); + rsi = (word64)(rsi + 0x80); + rbp = (word32)((word32)rbp - 0x80); + zf1 = (word32)rbp; + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rdi, 32) = (word64)(r12); + WC_S64(rdi, 40) = (word64)(r13); + WC_S64(rdi, 48) = (word64)(r14); + WC_S64(rdi, 56) = (word64)(r15); + if ((zf1) != (0)) { + goto L_sha512_len_avx1_rorx_begin; + } + rax = (word64)(0); + return (int)(word32)rax; +} + +#endif /* HAVE_INTEL_AVX1 */ +#ifdef HAVE_INTEL_AVX2 +XALIGNED(32) static const word64 L_avx2_sha512_k[] WC_X64I_UNUSED = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, +}; + +XALIGNED(32) static const word64 L_avx2_sha512_k_2[] WC_X64I_UNUSED = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, +}; + +XALIGNED(16) static const word64 L_avx2_sha512_k_2_end[] WC_X64I_UNUSED = { + (word64)(size_t)(L_avx2_sha512_k_2 + 128), +}; + +XALIGNED(32) static const word64 L_avx2_sha512_flip_mask[] WC_X64I_UNUSED = { + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL int Transform_Sha512_AVX2(wc_Sha512* sha512) +{ + word64 rdi, rsp, rax, r8, r9, r10, r11, r12, r13, r14, r15, rsi, rbx, + rdx = 0, rcx = 0; + __m256i y0, y1, y2, y3, y8, y9, y10 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(), y14 = _mm256_setzero_si256(), y15; + XALIGNED(32) WC_X64I_SLOT stk[20]; + + rdi = (word64)(size_t)sha512; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 136); + rax = (word64)(rdi + 64); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_sha512_flip_mask, 0)); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 32)); + y0 = _mm256_shuffle_epi8(y0, y15); + y1 = _mm256_shuffle_epi8(y1, y15); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 96)); + y2 = _mm256_shuffle_epi8(y2, y15); + y3 = _mm256_shuffle_epi8(y3, y15); + WC_S32(rsp, 128) = (word32)(4); + rsi = (word64)((word64)(size_t)L_avx2_sha512_k); + rbx = (word64)(r9); + rax = (word64)(r12); + rbx = (word64)(rbx ^ r10); + y8 = _mm256_add_epi64(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y9 = _mm256_add_epi64(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 32), y9); + y8 = _mm256_add_epi64(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y9 = _mm256_add_epi64(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 96), y9); + /* Start of 16 rounds */ +L_sha256_avx2_start: + rsi = (word64)(rsi + 0x80); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_blend_epi32(y0, y1, 3); + y13 = _mm256_blend_epi32(y2, y3, 3); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + y12 = _mm256_permute4x64_epi64(y12, 0x39); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + y13 = _mm256_permute4x64_epi64(y13, 0x39); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + y8 = _mm256_srli_epi64(y12, 1); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + y9 = _mm256_slli_epi64(y12, 63); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y10 = _mm256_srli_epi64(y12, 8); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + y8 = _mm256_or_si256(y8, y9); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + y10 = _mm256_or_si256(y10, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y11 = _mm256_srli_epi64(y12, 7); + rbx = (word64)(r15); + rcx = (word64)(r12); + y8 = _mm256_xor_si256(y8, y10); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rcx = (word64)(rcx ^ r13); + y8 = _mm256_xor_si256(y8, y11); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + y0 = _mm256_add_epi64(y13, y0); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + y0 = _mm256_add_epi64(y8, y0); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + y14 = _mm256_permute2x128_si256(y3, y3, 0x81); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y8 = _mm256_srli_epi64(y14, 19); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + y10 = _mm256_srli_epi64(y14, 61); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + y11 = _mm256_slli_epi64(y14, 3); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y8 = _mm256_or_si256(y8, y9); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rcx = (word64)(rcx ^ r12); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + y11 = _mm256_srli_epi64(y14, 6); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + y8 = _mm256_xor_si256(y8, y11); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + y0 = _mm256_add_epi64(y8, y0); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y14 = _mm256_permute2x128_si256(y0, y0, 8); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + y8 = _mm256_srli_epi64(y14, 19); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y10 = _mm256_srli_epi64(y14, 61); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ r11); + y11 = _mm256_slli_epi64(y14, 3); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + y8 = _mm256_or_si256(y8, y9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + y11 = _mm256_srli_epi64(y14, 6); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y0 = _mm256_add_epi64(y8, y0); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_blend_epi32(y1, y2, 3); + y13 = _mm256_blend_epi32(y3, y0, 3); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + y12 = _mm256_permute4x64_epi64(y12, 0x39); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + y13 = _mm256_permute4x64_epi64(y13, 0x39); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + y8 = _mm256_srli_epi64(y12, 1); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + y9 = _mm256_slli_epi64(y12, 63); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y10 = _mm256_srli_epi64(y12, 8); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + y8 = _mm256_or_si256(y8, y9); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + y10 = _mm256_or_si256(y10, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y11 = _mm256_srli_epi64(y12, 7); + rbx = (word64)(r11); + rcx = (word64)(r8); + y8 = _mm256_xor_si256(y8, y10); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rcx = (word64)(rcx ^ r9); + y8 = _mm256_xor_si256(y8, y11); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + y1 = _mm256_add_epi64(y13, y1); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + y1 = _mm256_add_epi64(y8, y1); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + y14 = _mm256_permute2x128_si256(y0, y0, 0x81); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y8 = _mm256_srli_epi64(y14, 19); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + y10 = _mm256_srli_epi64(y14, 61); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + y11 = _mm256_slli_epi64(y14, 3); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y8 = _mm256_or_si256(y8, y9); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rcx = (word64)(rcx ^ r8); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + y11 = _mm256_srli_epi64(y14, 6); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + y8 = _mm256_xor_si256(y8, y11); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + y1 = _mm256_add_epi64(y8, y1); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y14 = _mm256_permute2x128_si256(y1, y1, 8); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + y8 = _mm256_srli_epi64(y14, 19); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y10 = _mm256_srli_epi64(y14, 61); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rcx = (word64)(rcx ^ r15); + y11 = _mm256_slli_epi64(y14, 3); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + y8 = _mm256_or_si256(y8, y9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + y11 = _mm256_srli_epi64(y14, 6); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y1 = _mm256_add_epi64(y8, y1); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_blend_epi32(y2, y3, 3); + y13 = _mm256_blend_epi32(y0, y1, 3); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + y12 = _mm256_permute4x64_epi64(y12, 0x39); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + y13 = _mm256_permute4x64_epi64(y13, 0x39); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + y8 = _mm256_srli_epi64(y12, 1); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + y9 = _mm256_slli_epi64(y12, 63); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y10 = _mm256_srli_epi64(y12, 8); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + y8 = _mm256_or_si256(y8, y9); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + y10 = _mm256_or_si256(y10, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y11 = _mm256_srli_epi64(y12, 7); + rbx = (word64)(r15); + rcx = (word64)(r12); + y8 = _mm256_xor_si256(y8, y10); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rcx = (word64)(rcx ^ r13); + y8 = _mm256_xor_si256(y8, y11); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + y2 = _mm256_add_epi64(y13, y2); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + y2 = _mm256_add_epi64(y8, y2); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + y14 = _mm256_permute2x128_si256(y1, y1, 0x81); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y8 = _mm256_srli_epi64(y14, 19); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + y10 = _mm256_srli_epi64(y14, 61); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + y11 = _mm256_slli_epi64(y14, 3); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y8 = _mm256_or_si256(y8, y9); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rcx = (word64)(rcx ^ r12); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + y11 = _mm256_srli_epi64(y14, 6); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + y8 = _mm256_xor_si256(y8, y11); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + y2 = _mm256_add_epi64(y8, y2); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y14 = _mm256_permute2x128_si256(y2, y2, 8); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + y8 = _mm256_srli_epi64(y14, 19); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y10 = _mm256_srli_epi64(y14, 61); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rcx = (word64)(rcx ^ r11); + y11 = _mm256_slli_epi64(y14, 3); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + y8 = _mm256_or_si256(y8, y9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + y11 = _mm256_srli_epi64(y14, 6); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y2 = _mm256_add_epi64(y8, y2); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_blend_epi32(y3, y0, 3); + y13 = _mm256_blend_epi32(y1, y2, 3); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + y12 = _mm256_permute4x64_epi64(y12, 0x39); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + y13 = _mm256_permute4x64_epi64(y13, 0x39); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + y8 = _mm256_srli_epi64(y12, 1); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + y9 = _mm256_slli_epi64(y12, 63); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y10 = _mm256_srli_epi64(y12, 8); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + y8 = _mm256_or_si256(y8, y9); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + y10 = _mm256_or_si256(y10, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y11 = _mm256_srli_epi64(y12, 7); + rbx = (word64)(r11); + rcx = (word64)(r8); + y8 = _mm256_xor_si256(y8, y10); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rcx = (word64)(rcx ^ r9); + y8 = _mm256_xor_si256(y8, y11); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + y3 = _mm256_add_epi64(y13, y3); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + y3 = _mm256_add_epi64(y8, y3); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + y14 = _mm256_permute2x128_si256(y2, y2, 0x81); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y8 = _mm256_srli_epi64(y14, 19); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + y10 = _mm256_srli_epi64(y14, 61); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + y11 = _mm256_slli_epi64(y14, 3); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y8 = _mm256_or_si256(y8, y9); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rcx = (word64)(rcx ^ r8); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + y11 = _mm256_srli_epi64(y14, 6); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + y8 = _mm256_xor_si256(y8, y11); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + y3 = _mm256_add_epi64(y8, y3); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y14 = _mm256_permute2x128_si256(y3, y3, 8); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + y8 = _mm256_srli_epi64(y14, 19); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y10 = _mm256_srli_epi64(y14, 61); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rcx = (word64)(rcx ^ r15); + y11 = _mm256_slli_epi64(y14, 3); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + y8 = _mm256_or_si256(y8, y9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + y11 = _mm256_srli_epi64(y14, 6); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y3 = _mm256_add_epi64(y8, y3); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + y8 = _mm256_add_epi64(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y9 = _mm256_add_epi64(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 32), y9); + y8 = _mm256_add_epi64(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y9 = _mm256_add_epi64(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 96), y9); + WC_S32(rsp, 128) = (word32)(WC_L32(rsp, 128) - 1); + if (((word32)WC_S32(rsp, 128)) != (0)) { + goto L_sha256_avx2_start; + } + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) + r8); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) + r9); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) + r10); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) + r11); + WC_S64(rdi, 32) = (word64)(WC_L64(rdi, 32) + r12); + WC_S64(rdi, 40) = (word64)(WC_L64(rdi, 40) + r13); + WC_S64(rdi, 48) = (word64)(WC_L64(rdi, 48) + r14); + WC_S64(rdi, 56) = (word64)(WC_L64(rdi, 56) + r15); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL int Transform_Sha512_AVX2_Len(wc_Sha512* sha512, word32 len) +{ + word64 rdi, rbp, rbx = 0, rsp, rcx = 0, r8 = 0, r9 = 0, r10 = 0, r11 = 0, + r12 = 0, r13 = 0, r14 = 0, r15 = 0, rsi = 0, rax = 0, rdx = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y15 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[172]; + word32 zf1; + + rdi = (word64)(size_t)sha512; + rbp = (word64)(word32)len; + + rsp = (word64)(size_t)stk + 1376; + if ((((byte)rbp & 0x80)) == (0)) { + goto L_sha512_len_avx2_block; + } + rbx = (word64)(WC_L64(rdi, 224)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y3); + (void)Transform_Sha512_AVX2((wc_Sha512*)(size_t)rdi); + WC_S64(rdi, 224) = (word64)(WC_L64(rdi, 224) + 0x80); + rbp = (word32)((word32)rbp - 0x80); + if (((word32)rbp) == (0)) { + goto L_sha512_len_avx2_done; + } +L_sha512_len_avx2_block: + rsp = (word64)(rsp - 1352); + rcx = (word64)(WC_L64(rdi, 224)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_sha512_flip_mask, 0)); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + WC_S64(rsp, 1344) = (word64)(rbp); + /* Start of loop processing two blocks */ +L_sha512_len_avx2_begin: + rbp = (word64)(rsp); + rsi = (word64)((word64)(size_t)L_avx2_sha512_k_2); + rbx = (word64)(r9); + rax = (word64)(r12); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, 0))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 16))); + y0 = _mm256_inserti128_si256(y0, _mm_loadu_si128((const __m128i*)WC_PR(rcx, + 128)), 1); + y1 = _mm256_inserti128_si256(y1, _mm_loadu_si128((const __m128i*)WC_PR(rcx, + 144)), 1); + y0 = _mm256_shuffle_epi8(y0, y15); + y1 = _mm256_shuffle_epi8(y1, y15); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 32))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 48))); + y2 = _mm256_inserti128_si256(y2, _mm_loadu_si128((const __m128i*)WC_PR(rcx, + 160)), 1); + y3 = _mm256_inserti128_si256(y3, _mm_loadu_si128((const __m128i*)WC_PR(rcx, + 176)), 1); + y2 = _mm256_shuffle_epi8(y2, y15); + y3 = _mm256_shuffle_epi8(y3, y15); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 64))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 80))); + y4 = _mm256_inserti128_si256(y4, _mm_loadu_si128((const __m128i*)WC_PR(rcx, + 192)), 1); + y5 = _mm256_inserti128_si256(y5, _mm_loadu_si128((const __m128i*)WC_PR(rcx, + 208)), 1); + y4 = _mm256_shuffle_epi8(y4, y15); + y5 = _mm256_shuffle_epi8(y5, y15); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 96))); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rcx, + 112))); + y6 = _mm256_inserti128_si256(y6, _mm_loadu_si128((const __m128i*)WC_PR(rcx, + 224)), 1); + y7 = _mm256_inserti128_si256(y7, _mm_loadu_si128((const __m128i*)WC_PR(rcx, + 240)), 1); + y6 = _mm256_shuffle_epi8(y6, y15); + y7 = _mm256_shuffle_epi8(y7, y15); + rbx = (word64)(rbx ^ r10); + /* Start of 16 rounds */ +L_sha512_len_avx2_start: + y8 = _mm256_add_epi64(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y9 = _mm256_add_epi64(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 32), y9); + y8 = _mm256_add_epi64(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y9 = _mm256_add_epi64(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 96), y9); + y8 = _mm256_add_epi64(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + y9 = _mm256_add_epi64(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 128), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 160), y9); + y8 = _mm256_add_epi64(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + y9 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 192), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 224), y9); + /* msg_sched: 0-1 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_alignr_epi8(y1, y0, 8); + y13 = _mm256_alignr_epi8(y5, y4, 8); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rbp, 0)); + rcx = (word64)(rcx ^ r14); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + y8 = _mm256_xor_si256(y8, y11); + y0 = _mm256_add_epi64(y13, y0); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y0 = _mm256_add_epi64(y8, y0); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rbp, 8)); + rcx = (word64)(rcx ^ r13); + y8 = _mm256_srli_epi64(y7, 19); + y9 = _mm256_slli_epi64(y7, 45); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + y10 = _mm256_srli_epi64(y7, 61); + y11 = _mm256_slli_epi64(y7, 3); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y7, 6); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + y8 = _mm256_xor_si256(y8, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + y0 = _mm256_add_epi64(y8, y0); + /* msg_sched done: 0-1 */ + /* msg_sched: 4-5 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_alignr_epi8(y2, y1, 8); + y13 = _mm256_alignr_epi8(y6, y5, 8); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rbp, 32)); + rcx = (word64)(rcx ^ r12); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + y8 = _mm256_xor_si256(y8, y11); + y1 = _mm256_add_epi64(y13, y1); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y1 = _mm256_add_epi64(y8, y1); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rbp, 40)); + rcx = (word64)(rcx ^ r11); + y8 = _mm256_srli_epi64(y0, 19); + y9 = _mm256_slli_epi64(y0, 45); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + y10 = _mm256_srli_epi64(y0, 61); + y11 = _mm256_slli_epi64(y0, 3); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y0, 6); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + y8 = _mm256_xor_si256(y8, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + y1 = _mm256_add_epi64(y8, y1); + /* msg_sched done: 4-5 */ + /* msg_sched: 8-9 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_alignr_epi8(y3, y2, 8); + y13 = _mm256_alignr_epi8(y7, y6, 8); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rbp, 64)); + rcx = (word64)(rcx ^ r10); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + y8 = _mm256_xor_si256(y8, y11); + y2 = _mm256_add_epi64(y13, y2); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y2 = _mm256_add_epi64(y8, y2); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rbp, 72)); + rcx = (word64)(rcx ^ r9); + y8 = _mm256_srli_epi64(y1, 19); + y9 = _mm256_slli_epi64(y1, 45); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + y10 = _mm256_srli_epi64(y1, 61); + y11 = _mm256_slli_epi64(y1, 3); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y1, 6); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + y8 = _mm256_xor_si256(y8, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + y2 = _mm256_add_epi64(y8, y2); + /* msg_sched done: 8-9 */ + /* msg_sched: 12-13 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_alignr_epi8(y4, y3, 8); + y13 = _mm256_alignr_epi8(y0, y7, 8); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rbp, 96)); + rcx = (word64)(rcx ^ r8); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + y8 = _mm256_xor_si256(y8, y11); + y3 = _mm256_add_epi64(y13, y3); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y3 = _mm256_add_epi64(y8, y3); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rbp, 104)); + rcx = (word64)(rcx ^ r15); + y8 = _mm256_srli_epi64(y2, 19); + y9 = _mm256_slli_epi64(y2, 45); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + y10 = _mm256_srli_epi64(y2, 61); + y11 = _mm256_slli_epi64(y2, 3); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y2, 6); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + y8 = _mm256_xor_si256(y8, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + y3 = _mm256_add_epi64(y8, y3); + /* msg_sched done: 12-13 */ + /* msg_sched: 16-17 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_alignr_epi8(y5, y4, 8); + y13 = _mm256_alignr_epi8(y1, y0, 8); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rbp, 128)); + rcx = (word64)(rcx ^ r14); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + y8 = _mm256_xor_si256(y8, y11); + y4 = _mm256_add_epi64(y13, y4); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y4 = _mm256_add_epi64(y8, y4); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rbp, 136)); + rcx = (word64)(rcx ^ r13); + y8 = _mm256_srli_epi64(y3, 19); + y9 = _mm256_slli_epi64(y3, 45); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + y10 = _mm256_srli_epi64(y3, 61); + y11 = _mm256_slli_epi64(y3, 3); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y3, 6); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + y8 = _mm256_xor_si256(y8, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + y4 = _mm256_add_epi64(y8, y4); + /* msg_sched done: 16-17 */ + /* msg_sched: 20-21 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_alignr_epi8(y6, y5, 8); + y13 = _mm256_alignr_epi8(y2, y1, 8); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rbp, 160)); + rcx = (word64)(rcx ^ r12); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + y8 = _mm256_xor_si256(y8, y11); + y5 = _mm256_add_epi64(y13, y5); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y5 = _mm256_add_epi64(y8, y5); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rbp, 168)); + rcx = (word64)(rcx ^ r11); + y8 = _mm256_srli_epi64(y4, 19); + y9 = _mm256_slli_epi64(y4, 45); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + y10 = _mm256_srli_epi64(y4, 61); + y11 = _mm256_slli_epi64(y4, 3); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y4, 6); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + y8 = _mm256_xor_si256(y8, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + y5 = _mm256_add_epi64(y8, y5); + /* msg_sched done: 20-21 */ + /* msg_sched: 24-25 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_alignr_epi8(y7, y6, 8); + y13 = _mm256_alignr_epi8(y3, y2, 8); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rbp, 192)); + rcx = (word64)(rcx ^ r10); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + y8 = _mm256_xor_si256(y8, y11); + y6 = _mm256_add_epi64(y13, y6); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y6 = _mm256_add_epi64(y8, y6); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rbp, 200)); + rcx = (word64)(rcx ^ r9); + y8 = _mm256_srli_epi64(y5, 19); + y9 = _mm256_slli_epi64(y5, 45); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + y10 = _mm256_srli_epi64(y5, 61); + y11 = _mm256_slli_epi64(y5, 3); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y5, 6); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + y8 = _mm256_xor_si256(y8, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + y6 = _mm256_add_epi64(y8, y6); + /* msg_sched done: 24-25 */ + /* msg_sched: 28-29 */ + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y12 = _mm256_alignr_epi8(y0, y7, 8); + y13 = _mm256_alignr_epi8(y4, y3, 8); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rbp, 224)); + rcx = (word64)(rcx ^ r8); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + y8 = _mm256_xor_si256(y8, y11); + y7 = _mm256_add_epi64(y13, y7); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + y7 = _mm256_add_epi64(y8, y7); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rbp, 232)); + rcx = (word64)(rcx ^ r15); + y8 = _mm256_srli_epi64(y6, 19); + y9 = _mm256_slli_epi64(y6, 45); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + y10 = _mm256_srli_epi64(y6, 61); + y11 = _mm256_slli_epi64(y6, 3); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y6, 6); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + y8 = _mm256_xor_si256(y8, y11); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + y7 = _mm256_add_epi64(y8, y7); + /* msg_sched done: 28-29 */ + rsi = (word64)(rsi + 0x100); + rbp = (word64)(rbp + 0x100); + if ((rsi) != (WC_L64(L_avx2_sha512_k_2_end, 0))) { + goto L_sha512_len_avx2_start; + } + y8 = _mm256_add_epi64(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y9 = _mm256_add_epi64(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 32), y9); + y8 = _mm256_add_epi64(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y9 = _mm256_add_epi64(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 96), y9); + y8 = _mm256_add_epi64(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + y9 = _mm256_add_epi64(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 128), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 160), y9); + y8 = _mm256_add_epi64(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + y9 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 192), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rbp, 224), y9); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rbp, 0)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rbp, 8)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rbp, 32)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rbp, 40)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rbp, 64)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rbp, 72)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rbp, 96)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rbp, 104)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rbp, 128)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rbp, 136)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rbp, 160)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rbp, 168)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rbp, 192)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rbp, 200)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rbp, 224)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rbp, 232)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + rbp = (word64)(rbp - 0x400); + r8 = (word64)(r8 + WC_L64(rdi, 0)); + r9 = (word64)(r9 + WC_L64(rdi, 8)); + r10 = (word64)(r10 + WC_L64(rdi, 16)); + r11 = (word64)(r11 + WC_L64(rdi, 24)); + r12 = (word64)(r12 + WC_L64(rdi, 32)); + r13 = (word64)(r13 + WC_L64(rdi, 40)); + r14 = (word64)(r14 + WC_L64(rdi, 48)); + r15 = (word64)(r15 + WC_L64(rdi, 56)); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rdi, 32) = (word64)(r12); + WC_S64(rdi, 40) = (word64)(r13); + WC_S64(rdi, 48) = (word64)(r14); + WC_S64(rdi, 56) = (word64)(r15); + rbx = (word64)(r9); + rax = (word64)(r12); + rbx = (word64)(rbx ^ r10); + rsi = (word64)(5); +L_sha512_len_avx2_tail: + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rbp, 16)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rbp, 24)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rbp, 48)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rbp, 56)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rbp, 80)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rbp, 88)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rbp, 112)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rbp, 120)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r8); + rcx = (word64)(r13); + r15 = (word64)(r15 + WC_L64(rbp, 144)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + rcx = (word64)(rcx & r12); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r14); + rax = (word64)(rax ^ r12); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r9); + r15 = (word64)(r15 + rax); + rcx = (word64)(r8); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r8); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r11 = (word64)(r11 + r15); + rcx = (word64)(rcx ^ r8); + r15 = (word64)(r15 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r11); + r15 = (word64)(r15 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r15); + rcx = (word64)(r12); + r14 = (word64)(r14 + WC_L64(rbp, 152)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + rcx = (word64)(rcx & r11); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r13); + rax = (word64)(rax ^ r11); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r8); + r14 = (word64)(r14 + rax); + rcx = (word64)(r15); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r15); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r10 = (word64)(r10 + r14); + rcx = (word64)(rcx ^ r15); + r14 = (word64)(r14 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r10); + r14 = (word64)(r14 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r14); + rcx = (word64)(r11); + r13 = (word64)(r13 + WC_L64(rbp, 176)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + rcx = (word64)(rcx & r10); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r12); + rax = (word64)(rax ^ r10); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r15); + r13 = (word64)(r13 + rax); + rcx = (word64)(r14); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r14); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r9 = (word64)(r9 + r13); + rcx = (word64)(rcx ^ r14); + r13 = (word64)(r13 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r9); + r13 = (word64)(r13 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r13); + rcx = (word64)(r10); + r12 = (word64)(r12 + WC_L64(rbp, 184)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + rcx = (word64)(rcx & r9); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r11); + rax = (word64)(rax ^ r9); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r14); + r12 = (word64)(r12 + rax); + rcx = (word64)(r13); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r13); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r8 = (word64)(r8 + r12); + rcx = (word64)(rcx ^ r13); + r12 = (word64)(r12 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r8); + r12 = (word64)(r12 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r12); + rcx = (word64)(r9); + r11 = (word64)(r11 + WC_L64(rbp, 208)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + rcx = (word64)(rcx & r8); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r10); + rax = (word64)(rax ^ r8); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r13); + r11 = (word64)(r11 + rax); + rcx = (word64)(r12); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r12); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r15 = (word64)(r15 + r11); + rcx = (word64)(rcx ^ r12); + r11 = (word64)(r11 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r15); + r11 = (word64)(r11 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r11); + rcx = (word64)(r8); + r10 = (word64)(r10 + WC_L64(rbp, 216)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + rcx = (word64)(rcx & r15); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r9); + rax = (word64)(rax ^ r15); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r12); + r10 = (word64)(r10 + rax); + rcx = (word64)(r11); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r11); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r14 = (word64)(r14 + r10); + rcx = (word64)(rcx ^ r11); + r10 = (word64)(r10 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r14); + r10 = (word64)(r10 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rdx = (word64)(r10); + rcx = (word64)(r15); + r9 = (word64)(r9 + WC_L64(rbp, 240)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + rcx = (word64)(rcx & r14); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r8); + rax = (word64)(rax ^ r14); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rdx = (word64)(rdx ^ r11); + r9 = (word64)(r9 + rax); + rcx = (word64)(r10); + rbx = (word64)(rbx & rdx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r10); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r13 = (word64)(r13 + r9); + rcx = (word64)(rcx ^ r10); + r9 = (word64)(r9 + rbx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r13); + r9 = (word64)(r9 + rcx); + rax = (word64)(((rax) >> 23) | ((rax) << 41)); + rbx = (word64)(r9); + rcx = (word64)(r14); + r8 = (word64)(r8 + WC_L64(rbp, 248)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + rcx = (word64)(rcx & r13); + rax = (word64)(((rax) >> 4) | ((rax) << 60)); + rcx = (word64)(rcx ^ r15); + rax = (word64)(rax ^ r13); + r8 = (word64)(r8 + rcx); + rax = (word64)(((rax) >> 14) | ((rax) << 50)); + rbx = (word64)(rbx ^ r10); + r8 = (word64)(r8 + rax); + rcx = (word64)(r9); + rdx = (word64)(rdx & rbx); + rcx = (word64)(((rcx) >> 5) | ((rcx) << 59)); + rcx = (word64)(rcx ^ r9); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(((rcx) >> 6) | ((rcx) << 58)); + r12 = (word64)(r12 + r8); + rcx = (word64)(rcx ^ r9); + r8 = (word64)(r8 + rdx); + rcx = (word64)(((rcx) >> 28) | ((rcx) << 36)); + rax = (word64)(r12); + r8 = (word64)(r8 + rcx); + rbp = (word64)(rbp + 0x100); + rsi = (word64)(rsi - 1); + if ((rsi) != (0)) { + goto L_sha512_len_avx2_tail; + } + r8 = (word64)(r8 + WC_L64(rdi, 0)); + r9 = (word64)(r9 + WC_L64(rdi, 8)); + r10 = (word64)(r10 + WC_L64(rdi, 16)); + r11 = (word64)(r11 + WC_L64(rdi, 24)); + r12 = (word64)(r12 + WC_L64(rdi, 32)); + r13 = (word64)(r13 + WC_L64(rdi, 40)); + r14 = (word64)(r14 + WC_L64(rdi, 48)); + r15 = (word64)(r15 + WC_L64(rdi, 56)); + rcx = (word64)(WC_L64(rdi, 224)); + rcx = (word64)(rcx + 0x100); + WC_S32(rsp, 1344) = (word32)(WC_L32(rsp, 1344) - 0x100); + zf1 = (word32)WC_S32(rsp, 1344); + WC_S64(rdi, 224) = (word64)(rcx); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rdi, 32) = (word64)(r12); + WC_S64(rdi, 40) = (word64)(r13); + WC_S64(rdi, 48) = (word64)(r14); + WC_S64(rdi, 56) = (word64)(r15); + if ((zf1) != (0)) { + goto L_sha512_len_avx2_begin; + } + rsp = (word64)(rsp + 1352); +L_sha512_len_avx2_done: + rax = (word64)(0); + return (int)(word32)rax; +} + +XALIGNED(32) static const word64 L_avx2_rorx_sha512_k[] WC_X64I_UNUSED = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, +}; + +XALIGNED(32) static const word64 L_avx2_rorx_sha512_k_2[] WC_X64I_UNUSED = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, +}; + +XALIGNED(16) static const word64 L_avx2_rorx_sha512_k_2_end[] WC_X64I_UNUSED = { + (word64)(size_t)(L_avx2_rorx_sha512_k_2 + 128), +}; + +XALIGNED(32) static const word64 L_avx2_rorx_sha512_flip_mask[] + WC_X64I_UNUSED = { + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, + 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, +}; + +WC_X64I_TARGET("avx2,bmi2") +WOLFSSL_LOCAL int Transform_Sha512_AVX2_RORX(wc_Sha512* sha512) +{ + word64 rdi, rsp, rcx, r8, r9, r10, r11, r12, r13, r14, r15, rsi, rbx, rdx, + rax = 0; + __m256i y0, y1, y2, y3, y8, y9, y10 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(), y14 = _mm256_setzero_si256(), y15; + XALIGNED(32) WC_X64I_SLOT stk[20]; + + rdi = (word64)(size_t)sha512; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 136); + rcx = (word64)(rdi + 64); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_rorx_sha512_flip_mask, + 0)); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_shuffle_epi8(y0, y15); + y1 = _mm256_shuffle_epi8(y1, y15); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y2 = _mm256_shuffle_epi8(y2, y15); + y3 = _mm256_shuffle_epi8(y3, y15); + WC_S32(rsp, 128) = (word32)(4); + rsi = (word64)((word64)(size_t)L_avx2_rorx_sha512_k); + rbx = (word64)(r9); + rdx = (word64)(0); + rbx = (word64)(rbx ^ r10); + /* set_w_k: 0 */ + y8 = _mm256_add_epi64(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y9 = _mm256_add_epi64(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 32), y9); + y8 = _mm256_add_epi64(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y9 = _mm256_add_epi64(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 96), y9); + /* Start of 16 rounds */ +L_sha256_len_avx2_rorx_start: + rsi = (word64)(rsi + 0x80); + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + y12 = _mm256_blend_epi32(y0, y1, 3); + y13 = _mm256_blend_epi32(y2, y3, 3); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + y12 = _mm256_permute4x64_epi64(y12, 0x39); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + y13 = _mm256_permute4x64_epi64(y13, 0x39); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + y14 = _mm256_permute2x128_si256(y3, y3, 0x81); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + y8 = _mm256_xor_si256(y8, y10); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y11); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + y0 = _mm256_add_epi64(y13, y0); + y0 = _mm256_add_epi64(y8, y0); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + y8 = _mm256_srli_epi64(y14, 19); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y14, 61); + y11 = _mm256_slli_epi64(y14, 3); + y8 = _mm256_or_si256(y8, y9); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + y11 = _mm256_srli_epi64(y14, 6); + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + y8 = _mm256_xor_si256(y8, y11); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + y0 = _mm256_add_epi64(y8, y0); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + y14 = _mm256_permute2x128_si256(y0, y0, 8); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_srli_epi64(y14, 19); + y9 = _mm256_slli_epi64(y14, 45); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + y10 = _mm256_srli_epi64(y14, 61); + y11 = _mm256_slli_epi64(y14, 3); + y8 = _mm256_or_si256(y8, y9); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + y11 = _mm256_srli_epi64(y14, 6); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_xor_si256(y8, y11); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + y0 = _mm256_add_epi64(y8, y0); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + y8 = _mm256_add_epi64(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 0), y8); + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + y12 = _mm256_blend_epi32(y1, y2, 3); + y13 = _mm256_blend_epi32(y3, y0, 3); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + y12 = _mm256_permute4x64_epi64(y12, 0x39); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + y13 = _mm256_permute4x64_epi64(y13, 0x39); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + y14 = _mm256_permute2x128_si256(y0, y0, 0x81); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + y8 = _mm256_xor_si256(y8, y10); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y11); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + y1 = _mm256_add_epi64(y13, y1); + y1 = _mm256_add_epi64(y8, y1); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + y8 = _mm256_srli_epi64(y14, 19); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y14, 61); + y11 = _mm256_slli_epi64(y14, 3); + y8 = _mm256_or_si256(y8, y9); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + y11 = _mm256_srli_epi64(y14, 6); + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + y8 = _mm256_xor_si256(y8, y11); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + y1 = _mm256_add_epi64(y8, y1); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + y14 = _mm256_permute2x128_si256(y1, y1, 8); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_srli_epi64(y14, 19); + y9 = _mm256_slli_epi64(y14, 45); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + y10 = _mm256_srli_epi64(y14, 61); + y11 = _mm256_slli_epi64(y14, 3); + y8 = _mm256_or_si256(y8, y9); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + y11 = _mm256_srli_epi64(y14, 6); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_xor_si256(y8, y11); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + y1 = _mm256_add_epi64(y8, y1); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + y8 = _mm256_add_epi64(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 32), y8); + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + y12 = _mm256_blend_epi32(y2, y3, 3); + y13 = _mm256_blend_epi32(y0, y1, 3); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + y12 = _mm256_permute4x64_epi64(y12, 0x39); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + y13 = _mm256_permute4x64_epi64(y13, 0x39); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + y14 = _mm256_permute2x128_si256(y1, y1, 0x81); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + y8 = _mm256_xor_si256(y8, y10); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y11); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + y2 = _mm256_add_epi64(y13, y2); + y2 = _mm256_add_epi64(y8, y2); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + y8 = _mm256_srli_epi64(y14, 19); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y14, 61); + y11 = _mm256_slli_epi64(y14, 3); + y8 = _mm256_or_si256(y8, y9); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + y11 = _mm256_srli_epi64(y14, 6); + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + y8 = _mm256_xor_si256(y8, y11); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + y2 = _mm256_add_epi64(y8, y2); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + y14 = _mm256_permute2x128_si256(y2, y2, 8); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_srli_epi64(y14, 19); + y9 = _mm256_slli_epi64(y14, 45); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + y10 = _mm256_srli_epi64(y14, 61); + y11 = _mm256_slli_epi64(y14, 3); + y8 = _mm256_or_si256(y8, y9); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + y11 = _mm256_srli_epi64(y14, 6); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_xor_si256(y8, y11); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + y2 = _mm256_add_epi64(y8, y2); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + y8 = _mm256_add_epi64(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 64), y8); + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + y12 = _mm256_blend_epi32(y3, y0, 3); + y13 = _mm256_blend_epi32(y1, y2, 3); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + y12 = _mm256_permute4x64_epi64(y12, 0x39); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + y13 = _mm256_permute4x64_epi64(y13, 0x39); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + y14 = _mm256_permute2x128_si256(y2, y2, 0x81); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + y8 = _mm256_xor_si256(y8, y10); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y11); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + y3 = _mm256_add_epi64(y13, y3); + y3 = _mm256_add_epi64(y8, y3); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + y8 = _mm256_srli_epi64(y14, 19); + y9 = _mm256_slli_epi64(y14, 45); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y14, 61); + y11 = _mm256_slli_epi64(y14, 3); + y8 = _mm256_or_si256(y8, y9); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + y11 = _mm256_srli_epi64(y14, 6); + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + y8 = _mm256_xor_si256(y8, y11); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + y3 = _mm256_add_epi64(y8, y3); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + y14 = _mm256_permute2x128_si256(y3, y3, 8); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_srli_epi64(y14, 19); + y9 = _mm256_slli_epi64(y14, 45); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + y10 = _mm256_srli_epi64(y14, 61); + y11 = _mm256_slli_epi64(y14, 3); + y8 = _mm256_or_si256(y8, y9); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + y8 = _mm256_xor_si256(y8, y10); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + y11 = _mm256_srli_epi64(y14, 6); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_xor_si256(y8, y11); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + y3 = _mm256_add_epi64(y8, y3); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + y8 = _mm256_add_epi64(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + _mm256_storeu_si256((__m256i*)WC_PW(rsp, 96), y8); + WC_S32(rsp, 128) = (word32)(WC_L32(rsp, 128) - 1); + if (((word32)WC_S32(rsp, 128)) != (0)) { + goto L_sha256_len_avx2_rorx_start; + } + /* rnd_all_4: 0-3 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsp, 0)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsp, 8)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsp, 16)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsp, 24)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_4: 4-7 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsp, 32)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsp, 40)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsp, 48)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsp, 56)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + /* rnd_all_4: 8-11 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsp, 64)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsp, 72)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsp, 80)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsp, 88)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_4: 12-15 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsp, 96)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsp, 104)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsp, 112)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsp, 120)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + r8 = (word64)(r8 + rdx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) + r8); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) + r9); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) + r10); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) + r11); + WC_S64(rdi, 32) = (word64)(WC_L64(rdi, 32) + r12); + WC_S64(rdi, 40) = (word64)(WC_L64(rdi, 40) + r13); + WC_S64(rdi, 48) = (word64)(WC_L64(rdi, 48) + r14); + WC_S64(rdi, 56) = (word64)(WC_L64(rdi, 56) + r15); + rax = (word64)(0); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx2,bmi2") +WOLFSSL_LOCAL int Transform_Sha512_AVX2_RORX_Len(wc_Sha512* sha512, word32 len) +{ + word64 rdi, rsi, rax = 0, rsp, r8 = 0, r9 = 0, r10 = 0, r11 = 0, r12 = 0, + r13 = 0, r14 = 0, r15 = 0, rbp = 0, rbx = 0, rdx = 0, rcx = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), + y15 = _mm256_setzero_si256(); + XALIGNED(32) WC_X64I_SLOT stk[172]; + word32 zf1; + + rdi = (word64)(size_t)sha512; + rsi = (word64)(word32)len; + + rsp = (word64)(size_t)stk + 1376; + if ((((byte)rsi & 0x80)) == (0)) { + goto L_sha512_len_avx2_rorx_block; + } + rax = (word64)(WC_L64(rdi, 224)); + rsp = (word64)(rsp - 8); + WC_S64(rsp, 0) = rsi; + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 96)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y3); + (void)Transform_Sha512_AVX2_RORX((wc_Sha512*)(size_t)rdi); + rsi = WC_L64(rsp, 0); + rsp = (word64)(rsp + 8); + WC_S64(rdi, 224) = (word64)(WC_L64(rdi, 224) + 0x80); + rsi = (word32)((word32)rsi - 0x80); + if (((word32)rsi) == (0)) { + goto L_sha512_len_avx2_rorx_done; + } +L_sha512_len_avx2_rorx_block: + rsp = (word64)(rsp - 1352); + rax = (word64)(WC_L64(rdi, 224)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(L_avx2_rorx_sha512_flip_mask, + 0)); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + WC_S32(rsp, 1344) = (word32)((word32)rsi); + /* Start of loop processing two blocks */ +L_sha512_len_avx2_rorx_begin: + rsi = (word64)(rsp); + rbp = (word64)((word64)(size_t)L_avx2_rorx_sha512_k_2); + rbx = (word64)(r9); + rdx = (word64)(0); + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, 0))); + y1 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, + 16))); + y0 = _mm256_inserti128_si256(y0, _mm_loadu_si128((const __m128i*)WC_PR(rax, + 128)), 1); + y1 = _mm256_inserti128_si256(y1, _mm_loadu_si128((const __m128i*)WC_PR(rax, + 144)), 1); + y0 = _mm256_shuffle_epi8(y0, y15); + y1 = _mm256_shuffle_epi8(y1, y15); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, + 32))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, + 48))); + y2 = _mm256_inserti128_si256(y2, _mm_loadu_si128((const __m128i*)WC_PR(rax, + 160)), 1); + y3 = _mm256_inserti128_si256(y3, _mm_loadu_si128((const __m128i*)WC_PR(rax, + 176)), 1); + y2 = _mm256_shuffle_epi8(y2, y15); + y3 = _mm256_shuffle_epi8(y3, y15); + y4 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, + 64))); + y5 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, + 80))); + y4 = _mm256_inserti128_si256(y4, _mm_loadu_si128((const __m128i*)WC_PR(rax, + 192)), 1); + y5 = _mm256_inserti128_si256(y5, _mm_loadu_si128((const __m128i*)WC_PR(rax, + 208)), 1); + y4 = _mm256_shuffle_epi8(y4, y15); + y5 = _mm256_shuffle_epi8(y5, y15); + y6 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, + 96))); + y7 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rax, + 112))); + y6 = _mm256_inserti128_si256(y6, _mm_loadu_si128((const __m128i*)WC_PR(rax, + 224)), 1); + y7 = _mm256_inserti128_si256(y7, _mm_loadu_si128((const __m128i*)WC_PR(rax, + 240)), 1); + y6 = _mm256_shuffle_epi8(y6, y15); + y7 = _mm256_shuffle_epi8(y7, y15); + rbx = (word64)(rbx ^ r10); + /* Start of 16 rounds */ +L_sha512_len_avx2_rorx_start: + y8 = _mm256_add_epi64(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y9 = _mm256_add_epi64(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y9); + y8 = _mm256_add_epi64(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y9 = _mm256_add_epi64(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y9); + y8 = _mm256_add_epi64(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + y9 = _mm256_add_epi64(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y9); + y8 = _mm256_add_epi64(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + y9 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y9); + /* msg_sched: 0-1 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + y12 = _mm256_alignr_epi8(y1, y0, 8); + r15 = (word64)(r15 + WC_L64(rsi, 0)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + y13 = _mm256_alignr_epi8(y5, y4, 8); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + y8 = _mm256_xor_si256(y8, y11); + y0 = _mm256_add_epi64(y13, y0); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + y0 = _mm256_add_epi64(y8, y0); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + y8 = _mm256_srli_epi64(y7, 19); + y9 = _mm256_slli_epi64(y7, 45); + r14 = (word64)(r14 + WC_L64(rsi, 8)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y7, 61); + y11 = _mm256_slli_epi64(y7, 3); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y7, 6); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + y0 = _mm256_add_epi64(y8, y0); + /* msg_sched done: 0-1 */ + /* msg_sched: 4-5 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + y12 = _mm256_alignr_epi8(y2, y1, 8); + r13 = (word64)(r13 + WC_L64(rsi, 32)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + y13 = _mm256_alignr_epi8(y6, y5, 8); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + y8 = _mm256_xor_si256(y8, y11); + y1 = _mm256_add_epi64(y13, y1); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + y1 = _mm256_add_epi64(y8, y1); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + y8 = _mm256_srli_epi64(y0, 19); + y9 = _mm256_slli_epi64(y0, 45); + r12 = (word64)(r12 + WC_L64(rsi, 40)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y0, 61); + y11 = _mm256_slli_epi64(y0, 3); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y0, 6); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + y1 = _mm256_add_epi64(y8, y1); + /* msg_sched done: 4-5 */ + /* msg_sched: 8-9 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + y12 = _mm256_alignr_epi8(y3, y2, 8); + r11 = (word64)(r11 + WC_L64(rsi, 64)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + y13 = _mm256_alignr_epi8(y7, y6, 8); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + y8 = _mm256_xor_si256(y8, y11); + y2 = _mm256_add_epi64(y13, y2); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + y2 = _mm256_add_epi64(y8, y2); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + y8 = _mm256_srli_epi64(y1, 19); + y9 = _mm256_slli_epi64(y1, 45); + r10 = (word64)(r10 + WC_L64(rsi, 72)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y1, 61); + y11 = _mm256_slli_epi64(y1, 3); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y1, 6); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + y2 = _mm256_add_epi64(y8, y2); + /* msg_sched done: 8-9 */ + /* msg_sched: 12-13 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + y12 = _mm256_alignr_epi8(y4, y3, 8); + r9 = (word64)(r9 + WC_L64(rsi, 96)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + y13 = _mm256_alignr_epi8(y0, y7, 8); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + y8 = _mm256_xor_si256(y8, y11); + y3 = _mm256_add_epi64(y13, y3); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + y3 = _mm256_add_epi64(y8, y3); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + y8 = _mm256_srli_epi64(y2, 19); + y9 = _mm256_slli_epi64(y2, 45); + r8 = (word64)(r8 + WC_L64(rsi, 104)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y2, 61); + y11 = _mm256_slli_epi64(y2, 3); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y2, 6); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + y3 = _mm256_add_epi64(y8, y3); + /* msg_sched done: 12-13 */ + /* msg_sched: 16-17 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + y12 = _mm256_alignr_epi8(y5, y4, 8); + r15 = (word64)(r15 + WC_L64(rsi, 128)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + y13 = _mm256_alignr_epi8(y1, y0, 8); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + y8 = _mm256_xor_si256(y8, y11); + y4 = _mm256_add_epi64(y13, y4); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + y4 = _mm256_add_epi64(y8, y4); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + y8 = _mm256_srli_epi64(y3, 19); + y9 = _mm256_slli_epi64(y3, 45); + r14 = (word64)(r14 + WC_L64(rsi, 136)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y3, 61); + y11 = _mm256_slli_epi64(y3, 3); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y3, 6); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + y4 = _mm256_add_epi64(y8, y4); + /* msg_sched done: 16-17 */ + /* msg_sched: 20-21 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + y12 = _mm256_alignr_epi8(y6, y5, 8); + r13 = (word64)(r13 + WC_L64(rsi, 160)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + y13 = _mm256_alignr_epi8(y2, y1, 8); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + y8 = _mm256_xor_si256(y8, y11); + y5 = _mm256_add_epi64(y13, y5); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + y5 = _mm256_add_epi64(y8, y5); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + y8 = _mm256_srli_epi64(y4, 19); + y9 = _mm256_slli_epi64(y4, 45); + r12 = (word64)(r12 + WC_L64(rsi, 168)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y4, 61); + y11 = _mm256_slli_epi64(y4, 3); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y4, 6); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + y5 = _mm256_add_epi64(y8, y5); + /* msg_sched done: 20-21 */ + /* msg_sched: 24-25 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + y12 = _mm256_alignr_epi8(y7, y6, 8); + r11 = (word64)(r11 + WC_L64(rsi, 192)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + y13 = _mm256_alignr_epi8(y3, y2, 8); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + y8 = _mm256_xor_si256(y8, y11); + y6 = _mm256_add_epi64(y13, y6); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + y6 = _mm256_add_epi64(y8, y6); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + y8 = _mm256_srli_epi64(y5, 19); + y9 = _mm256_slli_epi64(y5, 45); + r10 = (word64)(r10 + WC_L64(rsi, 200)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y5, 61); + y11 = _mm256_slli_epi64(y5, 3); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y5, 6); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + y6 = _mm256_add_epi64(y8, y6); + /* msg_sched done: 24-25 */ + /* msg_sched: 28-29 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + y12 = _mm256_alignr_epi8(y0, y7, 8); + r9 = (word64)(r9 + WC_L64(rsi, 224)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + y13 = _mm256_alignr_epi8(y4, y3, 8); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_srli_epi64(y12, 1); + y9 = _mm256_slli_epi64(y12, 63); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + y10 = _mm256_srli_epi64(y12, 8); + y11 = _mm256_slli_epi64(y12, 56); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + y11 = _mm256_srli_epi64(y12, 7); + y8 = _mm256_xor_si256(y8, y10); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + y8 = _mm256_xor_si256(y8, y11); + y7 = _mm256_add_epi64(y13, y7); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + y7 = _mm256_add_epi64(y8, y7); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + y8 = _mm256_srli_epi64(y6, 19); + y9 = _mm256_slli_epi64(y6, 45); + r8 = (word64)(r8 + WC_L64(rsi, 232)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + y10 = _mm256_srli_epi64(y6, 61); + y11 = _mm256_slli_epi64(y6, 3); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + y8 = _mm256_or_si256(y8, y9); + y10 = _mm256_or_si256(y10, y11); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + y8 = _mm256_xor_si256(y8, y10); + y11 = _mm256_srli_epi64(y6, 6); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + y8 = _mm256_xor_si256(y8, y11); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + y7 = _mm256_add_epi64(y8, y7); + /* msg_sched done: 28-29 */ + rbp = (word64)(rbp + 0x100); + rsi = (word64)(rsi + 0x100); + if ((rbp) != (WC_L64(L_avx2_rorx_sha512_k_2_end, 0))) { + goto L_sha512_len_avx2_rorx_start; + } + y8 = _mm256_add_epi64(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + y9 = _mm256_add_epi64(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y9); + y8 = _mm256_add_epi64(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + y9 = _mm256_add_epi64(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y9); + y8 = _mm256_add_epi64(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + y9 = _mm256_add_epi64(y5, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y9); + y8 = _mm256_add_epi64(y6, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + y9 = _mm256_add_epi64(y7, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y9); + /* rnd_all_2: 0-1 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsi, 0)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsi, 8)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + /* rnd_all_2: 4-5 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsi, 32)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsi, 40)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_2: 8-9 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsi, 64)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsi, 72)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + /* rnd_all_2: 12-13 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsi, 96)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsi, 104)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + /* rnd_all_2: 16-17 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsi, 128)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsi, 136)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + /* rnd_all_2: 20-21 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsi, 160)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsi, 168)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_2: 24-25 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsi, 192)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsi, 200)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + /* rnd_all_2: 28-29 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsi, 224)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsi, 232)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + r8 = (word64)(r8 + rdx); + rsi = (word64)(rsi - 0x400); + r8 = (word64)(r8 + WC_L64(rdi, 0)); + r9 = (word64)(r9 + WC_L64(rdi, 8)); + r10 = (word64)(r10 + WC_L64(rdi, 16)); + r11 = (word64)(r11 + WC_L64(rdi, 24)); + r12 = (word64)(r12 + WC_L64(rdi, 32)); + r13 = (word64)(r13 + WC_L64(rdi, 40)); + r14 = (word64)(r14 + WC_L64(rdi, 48)); + r15 = (word64)(r15 + WC_L64(rdi, 56)); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rdi, 32) = (word64)(r12); + WC_S64(rdi, 40) = (word64)(r13); + WC_S64(rdi, 48) = (word64)(r14); + WC_S64(rdi, 56) = (word64)(r15); + rbx = (word64)(r9); + rdx = (word64)(0); + rbx = (word64)(rbx ^ r10); + rbp = (word64)(5); +L_sha512_len_avx2_rorx_tail: + /* rnd_all_2: 2-3 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsi, 16)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsi, 24)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + /* rnd_all_2: 6-7 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsi, 48)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsi, 56)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_2: 10-11 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsi, 80)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsi, 88)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + /* rnd_all_2: 14-15 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsi, 112)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsi, 120)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + /* rnd_all_2: 18-19 */ + rax = (word64)(((r12) >> 14) | ((r12) << 50)); + rcx = (word64)(((r12) >> 18) | ((r12) << 46)); + r8 = (word64)(r8 + rdx); + r15 = (word64)(r15 + WC_L64(rsi, 144)); + rdx = (word64)(r13); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r14); + rax = (word64)(((r12) >> 41) | ((r12) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r12); + r15 = (word64)(r15 + rax); + rax = (word64)(((r8) >> 28) | ((r8) << 36)); + rcx = (word64)(((r8) >> 34) | ((r8) << 30)); + rdx = (word64)(rdx ^ r14); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r8) >> 39) | ((r8) << 25)); + r15 = (word64)(r15 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r9); + r11 = (word64)(r11 + r15); + rdx = (word64)(rdx ^ r8); + rbx = (word64)(rbx & rdx); + r15 = (word64)(r15 + rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r11) >> 14) | ((r11) << 50)); + rcx = (word64)(((r11) >> 18) | ((r11) << 46)); + r15 = (word64)(r15 + rbx); + r14 = (word64)(r14 + WC_L64(rsi, 152)); + rbx = (word64)(r12); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r11) >> 41) | ((r11) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r11); + r14 = (word64)(r14 + rax); + rax = (word64)(((r15) >> 28) | ((r15) << 36)); + rcx = (word64)(((r15) >> 34) | ((r15) << 30)); + rbx = (word64)(rbx ^ r13); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r15) >> 39) | ((r15) << 25)); + r14 = (word64)(r14 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r8); + r10 = (word64)(r10 + r14); + rbx = (word64)(rbx ^ r15); + rdx = (word64)(rdx & rbx); + r14 = (word64)(r14 + rax); + rdx = (word64)(rdx ^ r8); + /* rnd_all_2: 22-23 */ + rax = (word64)(((r10) >> 14) | ((r10) << 50)); + rcx = (word64)(((r10) >> 18) | ((r10) << 46)); + r14 = (word64)(r14 + rdx); + r13 = (word64)(r13 + WC_L64(rsi, 176)); + rdx = (word64)(r11); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r12); + rax = (word64)(((r10) >> 41) | ((r10) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r10); + r13 = (word64)(r13 + rax); + rax = (word64)(((r14) >> 28) | ((r14) << 36)); + rcx = (word64)(((r14) >> 34) | ((r14) << 30)); + rdx = (word64)(rdx ^ r12); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r14) >> 39) | ((r14) << 25)); + r13 = (word64)(r13 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r15); + r9 = (word64)(r9 + r13); + rdx = (word64)(rdx ^ r14); + rbx = (word64)(rbx & rdx); + r13 = (word64)(r13 + rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r9) >> 14) | ((r9) << 50)); + rcx = (word64)(((r9) >> 18) | ((r9) << 46)); + r13 = (word64)(r13 + rbx); + r12 = (word64)(r12 + WC_L64(rsi, 184)); + rbx = (word64)(r10); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r9) >> 41) | ((r9) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r9); + r12 = (word64)(r12 + rax); + rax = (word64)(((r13) >> 28) | ((r13) << 36)); + rcx = (word64)(((r13) >> 34) | ((r13) << 30)); + rbx = (word64)(rbx ^ r11); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r13) >> 39) | ((r13) << 25)); + r12 = (word64)(r12 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r14); + r8 = (word64)(r8 + r12); + rbx = (word64)(rbx ^ r13); + rdx = (word64)(rdx & rbx); + r12 = (word64)(r12 + rax); + rdx = (word64)(rdx ^ r14); + /* rnd_all_2: 26-27 */ + rax = (word64)(((r8) >> 14) | ((r8) << 50)); + rcx = (word64)(((r8) >> 18) | ((r8) << 46)); + r12 = (word64)(r12 + rdx); + r11 = (word64)(r11 + WC_L64(rsi, 208)); + rdx = (word64)(r9); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r10); + rax = (word64)(((r8) >> 41) | ((r8) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r8); + r11 = (word64)(r11 + rax); + rax = (word64)(((r12) >> 28) | ((r12) << 36)); + rcx = (word64)(((r12) >> 34) | ((r12) << 30)); + rdx = (word64)(rdx ^ r10); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r12) >> 39) | ((r12) << 25)); + r11 = (word64)(r11 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r13); + r15 = (word64)(r15 + r11); + rdx = (word64)(rdx ^ r12); + rbx = (word64)(rbx & rdx); + r11 = (word64)(r11 + rax); + rbx = (word64)(rbx ^ r13); + rax = (word64)(((r15) >> 14) | ((r15) << 50)); + rcx = (word64)(((r15) >> 18) | ((r15) << 46)); + r11 = (word64)(r11 + rbx); + r10 = (word64)(r10 + WC_L64(rsi, 216)); + rbx = (word64)(r8); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r9); + rax = (word64)(((r15) >> 41) | ((r15) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r15); + r10 = (word64)(r10 + rax); + rax = (word64)(((r11) >> 28) | ((r11) << 36)); + rcx = (word64)(((r11) >> 34) | ((r11) << 30)); + rbx = (word64)(rbx ^ r9); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r11) >> 39) | ((r11) << 25)); + r10 = (word64)(r10 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r12); + r14 = (word64)(r14 + r10); + rbx = (word64)(rbx ^ r11); + rdx = (word64)(rdx & rbx); + r10 = (word64)(r10 + rax); + rdx = (word64)(rdx ^ r12); + /* rnd_all_2: 30-31 */ + rax = (word64)(((r14) >> 14) | ((r14) << 50)); + rcx = (word64)(((r14) >> 18) | ((r14) << 46)); + r10 = (word64)(r10 + rdx); + r9 = (word64)(r9 + WC_L64(rsi, 240)); + rdx = (word64)(r15); + rcx = (word64)(rcx ^ rax); + rdx = (word64)(rdx ^ r8); + rax = (word64)(((r14) >> 41) | ((r14) << 23)); + rax = (word64)(rax ^ rcx); + rdx = (word64)(rdx & r14); + r9 = (word64)(r9 + rax); + rax = (word64)(((r10) >> 28) | ((r10) << 36)); + rcx = (word64)(((r10) >> 34) | ((r10) << 30)); + rdx = (word64)(rdx ^ r8); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r10) >> 39) | ((r10) << 25)); + r9 = (word64)(r9 + rdx); + rax = (word64)(rax ^ rcx); + rdx = (word64)(r11); + r13 = (word64)(r13 + r9); + rdx = (word64)(rdx ^ r10); + rbx = (word64)(rbx & rdx); + r9 = (word64)(r9 + rax); + rbx = (word64)(rbx ^ r11); + rax = (word64)(((r13) >> 14) | ((r13) << 50)); + rcx = (word64)(((r13) >> 18) | ((r13) << 46)); + r9 = (word64)(r9 + rbx); + r8 = (word64)(r8 + WC_L64(rsi, 248)); + rbx = (word64)(r14); + rcx = (word64)(rcx ^ rax); + rbx = (word64)(rbx ^ r15); + rax = (word64)(((r13) >> 41) | ((r13) << 23)); + rax = (word64)(rax ^ rcx); + rbx = (word64)(rbx & r13); + r8 = (word64)(r8 + rax); + rax = (word64)(((r9) >> 28) | ((r9) << 36)); + rcx = (word64)(((r9) >> 34) | ((r9) << 30)); + rbx = (word64)(rbx ^ r15); + rcx = (word64)(rcx ^ rax); + rax = (word64)(((r9) >> 39) | ((r9) << 25)); + r8 = (word64)(r8 + rbx); + rax = (word64)(rax ^ rcx); + rbx = (word64)(r10); + r12 = (word64)(r12 + r8); + rbx = (word64)(rbx ^ r9); + rdx = (word64)(rdx & rbx); + r8 = (word64)(r8 + rax); + rdx = (word64)(rdx ^ r10); + rsi = (word64)(rsi + 0x100); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_sha512_len_avx2_rorx_tail; + } + r8 = (word64)(r8 + rdx); + r8 = (word64)(r8 + WC_L64(rdi, 0)); + r9 = (word64)(r9 + WC_L64(rdi, 8)); + r10 = (word64)(r10 + WC_L64(rdi, 16)); + r11 = (word64)(r11 + WC_L64(rdi, 24)); + r12 = (word64)(r12 + WC_L64(rdi, 32)); + r13 = (word64)(r13 + WC_L64(rdi, 40)); + r14 = (word64)(r14 + WC_L64(rdi, 48)); + r15 = (word64)(r15 + WC_L64(rdi, 56)); + rax = (word64)(WC_L64(rdi, 224)); + rax = (word64)(rax + 0x100); + WC_S32(rsp, 1344) = (word32)(WC_L32(rsp, 1344) - 0x100); + zf1 = (word32)WC_S32(rsp, 1344); + WC_S64(rdi, 224) = (word64)(rax); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + WC_S64(rdi, 32) = (word64)(r12); + WC_S64(rdi, 40) = (word64)(r13); + WC_S64(rdi, 48) = (word64)(r14); + WC_S64(rdi, 56) = (word64)(r15); + if ((zf1) != (0)) { + goto L_sha512_len_avx2_rorx_begin; + } + rsp = (word64)(rsp + 1352); +L_sha512_len_avx2_rorx_done: + rax = (word64)(0); + return (int)(word32)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/sp_x86_64_asm.S b/wolfcrypt/src/sp_x86_64_asm.S index 3994867b4e0..61ca591a626 100644 --- a/wolfcrypt/src/sp_x86_64_asm.S +++ b/wolfcrypt/src/sp_x86_64_asm.S @@ -45182,7 +45182,7 @@ _sp_256_mont_tpl_4: movq $0xffffffff00000001, %r10 adcq 16(%rsi), %rcx adcq 24(%rsi), %r8 - sbbq $0x00, %r11 + sbbq %r11, %r11 movl %r11d, %r9d andq %r11, %r10 subq %r11, %rdx @@ -45361,7 +45361,7 @@ _sp_256_mont_rsb_sub_dbl_4: movq $0xffffffff00000001, %r15 sbbq %r12, %r8 sbbq %r13, %r9 - sbbq $0x00, %rsi + sbbq %rsi, %rsi movl %esi, %r14d andq %rsi, %r15 addq %rsi, %rax diff --git a/wolfcrypt/src/sp_x86_64_asm.asm b/wolfcrypt/src/sp_x86_64_asm.asm index 690beda8957..f58813f6841 100644 --- a/wolfcrypt/src/sp_x86_64_asm.asm +++ b/wolfcrypt/src/sp_x86_64_asm.asm @@ -43853,7 +43853,7 @@ sp_256_mont_tpl_4 PROC mov r12, 18446744069414584321 adc r9, QWORD PTR [rdx+16] adc r10, QWORD PTR [rdx+24] - sbb r13, 0 + sbb r13, r13 mov r11d, r13d and r12, r13 sub rax, r13 @@ -44005,7 +44005,7 @@ sp_256_mont_rsb_sub_dbl_4 PROC mov rsi, 18446744069414584321 sbb r10, r14 sbb r11, r15 - sbb rdx, 0 + sbb rdx, rdx mov edi, edx and rsi, rdx add rax, rdx diff --git a/wolfcrypt/src/sp_x86_64_intrin.c b/wolfcrypt/src/sp_x86_64_intrin.c new file mode 100644 index 00000000000..d75ce50efd5 --- /dev/null +++ b/wolfcrypt/src/sp_x86_64_intrin.c @@ -0,0 +1,53134 @@ +/* sp_x86_64_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_SP_X86_64_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_SP_X86_64_ASM +#ifndef WOLFSSL_SP_NO_256 +/* Point structure to use. */ +typedef struct sp_point_256 { + /* X ordinate of point. */ + sp_digit x[2 * 4]; + /* Y ordinate of point. */ + sp_digit y[2 * 4]; + /* Z ordinate of point. */ + sp_digit z[2 * 4]; + /* Indicates point is at infinity. */ + int infinity; +} sp_point_256; +/* A table entry for pre-computed points. */ +typedef struct sp_table_entry_256 { + sp_digit x[4]; + sp_digit y[4]; +} sp_table_entry_256; + +#endif +#ifdef WOLFSSL_SP_384 +/* Point structure to use. */ +typedef struct sp_point_384 { + /* X ordinate of point. */ + sp_digit x[2 * 6]; + /* Y ordinate of point. */ + sp_digit y[2 * 6]; + /* Z ordinate of point. */ + sp_digit z[2 * 6]; + /* Indicates point is at infinity. */ + int infinity; +} sp_point_384; +/* A table entry for pre-computed points. */ +typedef struct sp_table_entry_384 { + sp_digit x[6]; + sp_digit y[6]; +} sp_table_entry_384; + +#endif +#ifdef WOLFSSL_SP_521 +/* Point structure to use. */ +typedef struct sp_point_521 { + /* X ordinate of point. */ + sp_digit x[2 * 9]; + /* Y ordinate of point. */ + sp_digit y[2 * 9]; + /* Z ordinate of point. */ + sp_digit z[2 * 9]; + /* Indicates point is at infinity. */ + int infinity; +#ifdef SP_ALIGN_16 + byte pad[16-sizeof(int)]; +#endif +} sp_point_521; +/* A table entry for pre-computed points. */ +typedef struct sp_table_entry_521 { + sp_digit x[9]; + sp_digit y[9]; +} sp_table_entry_521; + +#endif +#ifdef WOLFSSL_SP_1024 +/* Point structure to use. */ +typedef struct sp_point_1024 { + /* X ordinate of point. */ + sp_digit x[2 * 16]; + /* Y ordinate of point. */ + sp_digit y[2 * 16]; + /* Z ordinate of point. */ + sp_digit z[2 * 16]; + /* Indicates point is at infinity. */ + int infinity; +} sp_point_1024; +/* A table entry for pre-computed points. */ +typedef struct sp_table_entry_1024 { + sp_digit x[16]; + sp_digit y[16]; +} sp_table_entry_1024; + + +#endif +#ifndef WOLFSSL_SP_NO_2048 +#ifndef WOLFSSL_SP_NO_2048 +extern WOLFSSL_LOCAL void sp_2048_from_bin_bswap(sp_digit* r, int size, + const byte* a, int n); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_2048_from_bin_movbe(sp_digit* r, int size, + const byte* a, int n); +#endif +extern WOLFSSL_LOCAL void sp_2048_to_bin_bswap_32(sp_digit* r, byte* a); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_2048_to_bin_movbe_32(sp_digit* r, byte* a); +#endif +extern WOLFSSL_LOCAL void sp_2048_mul_16(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_2048_mul_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#endif +extern WOLFSSL_LOCAL sp_digit sp_2048_add_16(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_2048_sub_in_place_32(sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_2048_add_32(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_2048_mul_32(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_2048_mul_avx2_32(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#endif +extern WOLFSSL_LOCAL void sp_2048_sqr_16(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_2048_sqr_avx2_16(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL void sp_2048_sqr_32(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_2048_sqr_avx2_32(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL sp_digit sp_2048_sub_in_place_16(sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_2048_mul_d_32(sp_digit* r, const sp_digit* a, + sp_digit b); +extern WOLFSSL_LOCAL sp_digit sp_2048_cond_sub_16(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_2048_mont_reduce_16(sp_digit* a, const sp_digit* m, + sp_digit mp); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL sp_digit sp_2048_cond_sub_avx2_16(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#endif +extern WOLFSSL_LOCAL void sp_2048_mul_d_16(sp_digit* r, const sp_digit* a, + sp_digit b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_2048_mul_d_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit b); +#endif +#ifdef _WIN64 +extern WOLFSSL_LOCAL sp_digit div_2048_word_asm_16(sp_digit d1, sp_digit d0, + sp_digit div); +#endif +extern WOLFSSL_LOCAL sp_int64 sp_2048_cmp_16(const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_2048_mont_reduce_avx2_16(sp_digit* a, + const sp_digit* m, sp_digit mp); +#endif +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_2048_get_from_table_avx2_16(sp_digit* r, + sp_digit** table, int idx); +#endif +extern WOLFSSL_LOCAL sp_digit sp_2048_cond_sub_32(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_2048_mont_reduce_32(sp_digit* a, const sp_digit* m, + sp_digit mp); +extern WOLFSSL_LOCAL sp_digit sp_2048_sub_32(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_2048_mul_d_avx2_32(sp_digit* r, const sp_digit* a, + const sp_digit b); +#endif +#ifdef _WIN64 +extern WOLFSSL_LOCAL sp_digit div_2048_word_asm_32(sp_digit d1, sp_digit d0, + sp_digit div); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL sp_digit sp_2048_cond_sub_avx2_32(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#endif +extern WOLFSSL_LOCAL sp_int64 sp_2048_cmp_32(const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_2048_mont_reduce_avx2_32(sp_digit* a, + const sp_digit* m, sp_digit mp); +#endif +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_2048_get_from_table_avx2_32(sp_digit* r, + sp_digit** table, int idx); +#endif +extern WOLFSSL_LOCAL sp_digit sp_2048_cond_add_16(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL sp_digit sp_2048_cond_add_avx2_16(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#endif +extern WOLFSSL_LOCAL void sp_2048_lshift_32(sp_digit* r, const sp_digit* a, + int n); +#endif +#endif +#ifndef WOLFSSL_SP_NO_3072 +#ifndef WOLFSSL_SP_NO_3072 +extern WOLFSSL_LOCAL void sp_3072_from_bin_bswap(sp_digit* r, int size, + const byte* a, int n); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_3072_from_bin_movbe(sp_digit* r, int size, + const byte* a, int n); +#endif +extern WOLFSSL_LOCAL void sp_3072_to_bin_bswap_48(sp_digit* r, byte* a); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_3072_to_bin_movbe_48(sp_digit* r, byte* a); +#endif +extern WOLFSSL_LOCAL void sp_3072_mul_12(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_mul_avx2_12(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#endif +extern WOLFSSL_LOCAL sp_digit sp_3072_add_12(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_3072_sub_in_place_24(sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_3072_add_24(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_3072_mul_24(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_mul_avx2_24(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#endif +extern WOLFSSL_LOCAL sp_digit sp_3072_sub_in_place_48(sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_3072_add_48(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_3072_mul_48(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_mul_avx2_48(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#endif +extern WOLFSSL_LOCAL void sp_3072_sqr_12(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_sqr_avx2_12(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL void sp_3072_sqr_24(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_sqr_avx2_24(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL void sp_3072_sqr_48(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_sqr_avx2_48(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL void sp_3072_mul_d_48(sp_digit* r, const sp_digit* a, + sp_digit b); +extern WOLFSSL_LOCAL sp_digit sp_3072_cond_sub_24(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_3072_mont_reduce_24(sp_digit* a, const sp_digit* m, + sp_digit mp); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL sp_digit sp_3072_cond_sub_avx2_24(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#endif +extern WOLFSSL_LOCAL void sp_3072_mul_d_24(sp_digit* r, const sp_digit* a, + sp_digit b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_mul_d_avx2_24(sp_digit* r, const sp_digit* a, + const sp_digit b); +#endif +#ifdef _WIN64 +extern WOLFSSL_LOCAL sp_digit div_3072_word_asm_24(sp_digit d1, sp_digit d0, + sp_digit div); +#endif +extern WOLFSSL_LOCAL sp_int64 sp_3072_cmp_24(const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_mont_reduce_avx2_24(sp_digit* a, + const sp_digit* m, sp_digit mp); +#endif +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_3072_get_from_table_avx2_24(sp_digit* r, + sp_digit** table, int idx); +#endif +extern WOLFSSL_LOCAL sp_digit sp_3072_cond_sub_48(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_3072_mont_reduce_48(sp_digit* a, const sp_digit* m, + sp_digit mp); +extern WOLFSSL_LOCAL sp_digit sp_3072_sub_48(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_mul_d_avx2_48(sp_digit* r, const sp_digit* a, + const sp_digit b); +#endif +#ifdef _WIN64 +extern WOLFSSL_LOCAL sp_digit div_3072_word_asm_48(sp_digit d1, sp_digit d0, + sp_digit div); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL sp_digit sp_3072_cond_sub_avx2_48(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#endif +extern WOLFSSL_LOCAL sp_int64 sp_3072_cmp_48(const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_3072_mont_reduce_avx2_48(sp_digit* a, + const sp_digit* m, sp_digit mp); +#endif +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_3072_get_from_table_avx2_48(sp_digit* r, + sp_digit** table, int idx); +#endif +extern WOLFSSL_LOCAL sp_digit sp_3072_cond_add_24(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL sp_digit sp_3072_cond_add_avx2_24(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#endif +extern WOLFSSL_LOCAL void sp_3072_lshift_48(sp_digit* r, const sp_digit* a, + int n); +#endif +#endif +#ifdef WOLFSSL_SP_4096 +#ifdef WOLFSSL_SP_4096 +extern WOLFSSL_LOCAL void sp_4096_from_bin_bswap(sp_digit* r, int size, + const byte* a, int n); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_4096_from_bin_movbe(sp_digit* r, int size, + const byte* a, int n); +#endif +extern WOLFSSL_LOCAL void sp_4096_to_bin_bswap_64(sp_digit* r, byte* a); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_4096_to_bin_movbe_64(sp_digit* r, byte* a); +#endif +extern WOLFSSL_LOCAL sp_digit sp_4096_sub_in_place_64(sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_4096_add_64(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_4096_mul_64(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_4096_mul_avx2_64(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#endif +extern WOLFSSL_LOCAL void sp_4096_sqr_64(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_4096_sqr_avx2_64(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL void sp_4096_mul_d_64(sp_digit* r, const sp_digit* a, + sp_digit b); +extern WOLFSSL_LOCAL sp_digit sp_4096_cond_sub_64(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_4096_mont_reduce_64(sp_digit* a, const sp_digit* m, + sp_digit mp); +extern WOLFSSL_LOCAL sp_digit sp_4096_sub_64(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_4096_mul_d_avx2_64(sp_digit* r, const sp_digit* a, + const sp_digit b); +#endif +#ifdef _WIN64 +extern WOLFSSL_LOCAL sp_digit div_4096_word_asm_64(sp_digit d1, sp_digit d0, + sp_digit div); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL sp_digit sp_4096_cond_sub_avx2_64(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#endif +extern WOLFSSL_LOCAL sp_int64 sp_4096_cmp_64(const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_4096_mont_reduce_avx2_64(sp_digit* a, + const sp_digit* m, sp_digit mp); +#endif +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_4096_get_from_table_avx2_64(sp_digit* r, + sp_digit** table, int idx); +#endif +extern WOLFSSL_LOCAL sp_digit sp_4096_cond_add_32(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL sp_digit sp_4096_cond_add_avx2_32(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#endif +extern WOLFSSL_LOCAL void sp_4096_lshift_64(sp_digit* r, const sp_digit* a, + int n); +#endif +#endif +#ifndef WOLFSSL_SP_NO_256 +extern WOLFSSL_LOCAL void sp_256_mul_4(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_256_mul_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#endif +extern WOLFSSL_LOCAL void sp_256_sqr_4(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_256_sqr_avx2_4(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL sp_digit sp_256_add_4(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_256_sub_4(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_256_cond_copy_4(sp_digit* r, const sp_digit* a, + sp_digit m); +extern WOLFSSL_LOCAL void sp_256_mont_mul_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL void sp_256_mont_sqr_4(sp_digit* r, const sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL sp_int64 sp_256_cmp_4(const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_256_cond_sub_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_256_mont_reduce_4(sp_digit* a, const sp_digit* m, + sp_digit mp); +extern WOLFSSL_LOCAL void sp_256_mont_reduce_order_4(sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL void sp_256_mont_add_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_256_mont_dbl_4(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL void sp_256_mont_tpl_4(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL void sp_256_mont_sub_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_256_mont_div2_4(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL void sp_256_mont_rsb_sub_dbl_4(sp_digit* r, + const sp_digit* a, sp_digit* b, const sp_digit* m); +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_256_get_point_33_4(sp_point_256* r, + const sp_point_256* table, int idx); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_256_get_point_33_avx2_4(sp_point_256* r, + const sp_point_256* table, int idx); +#endif +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_256_mont_mul_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL void sp_256_mont_sqr_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL sp_digit sp_256_cond_sub_avx2_4(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_256_mont_reduce_order_avx2_4(sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL void sp_256_mont_div2_avx2_4(sp_digit* r, + const sp_digit* a, const sp_digit* m); +#endif +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_256_get_entry_64_4(sp_point_256* r, + const sp_table_entry_256* table, int idx); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_256_get_entry_64_avx2_4(sp_point_256* r, + const sp_table_entry_256* table, int idx); +#endif +extern WOLFSSL_LOCAL void sp_256_get_entry_65_4(sp_point_256* r, + const sp_table_entry_256* table, int idx); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_256_get_entry_65_avx2_4(sp_point_256* r, + const sp_table_entry_256* table, int idx); +#endif +#endif +extern WOLFSSL_LOCAL void sp_256_add_one_4(sp_digit* a); +extern WOLFSSL_LOCAL void sp_256_from_bin_bswap(sp_digit* r, int size, + const byte* a, int n); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_256_from_bin_movbe(sp_digit* r, int size, + const byte* a, int n); +#endif +extern WOLFSSL_LOCAL void sp_256_to_bin_bswap_4(sp_digit* r, byte* a); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_256_to_bin_movbe_4(sp_digit* r, byte* a); +#endif +extern WOLFSSL_LOCAL sp_digit sp_256_sub_in_place_4(sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_256_mul_d_4(sp_digit* r, const sp_digit* a, + sp_digit b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_256_mul_d_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit b); +#endif +#ifdef _WIN64 +extern WOLFSSL_LOCAL sp_digit div_256_word_asm_4(sp_digit d1, sp_digit d0, + sp_digit div); +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_256_mont_mul_order_avx2_4(sp_digit* r, + const sp_digit* a, const sp_digit* b); +extern WOLFSSL_LOCAL void sp_256_mont_sqr_order_avx2_4(sp_digit* r, + const sp_digit* a); +#endif +extern WOLFSSL_LOCAL void sp_256_mod_inv_4(sp_digit* r, const sp_digit* a, + const sp_digit* m); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_256_mod_inv_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* m); +#endif +#endif +#ifdef WOLFSSL_SP_384 +extern WOLFSSL_LOCAL void sp_384_mul_6(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_384_mul_avx2_6(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#endif +extern WOLFSSL_LOCAL void sp_384_sqr_6(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_384_sqr_avx2_6(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL sp_digit sp_384_add_6(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_384_sub_6(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_384_cond_copy_6(sp_digit* r, const sp_digit* a, + sp_digit m); +extern WOLFSSL_LOCAL sp_digit sp_384_cond_sub_6(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_384_mont_reduce_6(sp_digit* a, const sp_digit* m, + sp_digit mp); +extern WOLFSSL_LOCAL void sp_384_mont_reduce_order_6(sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL sp_int64 sp_384_cmp_6(const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_384_mont_add_6(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_384_mont_dbl_6(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL void sp_384_mont_tpl_6(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL void sp_384_mont_sub_6(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_384_mont_div2_6(sp_digit* r, const sp_digit* a, + const sp_digit* m); +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_384_get_point_33_6(sp_point_384* r, + const sp_point_384* table, int idx); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_384_get_point_33_avx2_6(sp_point_384* r, + const sp_point_384* table, int idx); +#endif +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_384_mont_reduce_order_avx2_6(sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL sp_digit sp_384_cond_sub_avx2_6(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_384_mont_div2_avx2_6(sp_digit* r, + const sp_digit* a, const sp_digit* m); +#endif +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_384_get_entry_64_6(sp_point_384* r, + const sp_table_entry_384* table, int idx); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_384_get_entry_64_avx2_6(sp_point_384* r, + const sp_table_entry_384* table, int idx); +#endif +extern WOLFSSL_LOCAL void sp_384_get_entry_65_6(sp_point_384* r, + const sp_table_entry_384* table, int idx); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_384_get_entry_65_avx2_6(sp_point_384* r, + const sp_table_entry_384* table, int idx); +#endif +#endif +extern WOLFSSL_LOCAL void sp_384_add_one_6(sp_digit* a); +extern WOLFSSL_LOCAL void sp_384_from_bin_bswap(sp_digit* r, int size, + const byte* a, int n); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_384_from_bin_movbe(sp_digit* r, int size, + const byte* a, int n); +#endif +extern WOLFSSL_LOCAL void sp_384_to_bin_bswap_6(sp_digit* r, byte* a); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_384_to_bin_movbe_6(sp_digit* r, byte* a); +#endif +extern WOLFSSL_LOCAL sp_digit sp_384_sub_in_place_6(sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_384_mul_d_6(sp_digit* r, const sp_digit* a, + sp_digit b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_384_mul_d_avx2_6(sp_digit* r, const sp_digit* a, + const sp_digit b); +#endif +#ifdef _WIN64 +extern WOLFSSL_LOCAL sp_digit div_384_word_asm_6(sp_digit d1, sp_digit d0, + sp_digit div); +#endif +extern WOLFSSL_LOCAL void sp_384_rshift1_6(sp_digit* r, const sp_digit* a); +extern WOLFSSL_LOCAL void sp_384_div2_mod_6(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL int sp_384_num_bits_6(const sp_digit * a); +#endif +#ifdef WOLFSSL_SP_521 +extern WOLFSSL_LOCAL void sp_521_mul_9(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_521_mul_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit* b); +#endif +extern WOLFSSL_LOCAL void sp_521_sqr_9(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_521_sqr_avx2_9(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL sp_digit sp_521_add_9(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_521_sub_9(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_521_cond_copy_9(sp_digit* r, const sp_digit* a, + sp_digit m); +extern WOLFSSL_LOCAL void sp_521_mont_mul_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL void sp_521_mont_sqr_9(sp_digit* r, const sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL sp_int64 sp_521_cmp_9(const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_521_cond_sub_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_521_mont_reduce_9(sp_digit* a, const sp_digit* m, + sp_digit mp); +extern WOLFSSL_LOCAL void sp_521_mont_reduce_order_9(sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL void sp_521_mont_add_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_521_mont_dbl_9(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL void sp_521_mont_tpl_9(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL void sp_521_mont_sub_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_521_mont_div2_9(sp_digit* r, const sp_digit* a, + const sp_digit* m); +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_521_get_point_33_9(sp_point_521* r, + const sp_point_521* table, int idx); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_521_get_point_33_avx2_9(sp_point_521* r, + const sp_point_521* table, int idx); +#endif +#endif +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_521_mont_mul_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL void sp_521_mont_sqr_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL sp_digit sp_521_cond_sub_avx2_9(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +extern WOLFSSL_LOCAL void sp_521_mont_reduce_order_avx2_9(sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL void sp_521_mont_div2_avx2_9(sp_digit* r, + const sp_digit* a, const sp_digit* m); +#endif +#ifndef WC_NO_CACHE_RESISTANT +extern WOLFSSL_LOCAL void sp_521_get_entry_64_9(sp_point_521* r, + const sp_table_entry_521* table, int idx); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_521_get_entry_64_avx2_9(sp_point_521* r, + const sp_table_entry_521* table, int idx); +#endif +extern WOLFSSL_LOCAL void sp_521_get_entry_65_9(sp_point_521* r, + const sp_table_entry_521* table, int idx); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_521_get_entry_65_avx2_9(sp_point_521* r, + const sp_table_entry_521* table, int idx); +#endif +#endif +extern WOLFSSL_LOCAL void sp_521_add_one_9(sp_digit* a); +extern WOLFSSL_LOCAL void sp_521_from_bin_bswap(sp_digit* r, int size, + const byte* a, int n); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_521_from_bin_movbe(sp_digit* r, int size, + const byte* a, int n); +#endif +extern WOLFSSL_LOCAL void sp_521_to_bin_bswap_9(sp_digit* r, byte* a); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_521_to_bin_movbe_9(sp_digit* r, byte* a); +#endif +extern WOLFSSL_LOCAL void sp_521_rshift_9(sp_digit* r, const sp_digit* a, + int n); +extern WOLFSSL_LOCAL void sp_521_lshift_9(sp_digit* r, const sp_digit* a, + int n); +extern WOLFSSL_LOCAL void sp_521_lshift_18(sp_digit* r, const sp_digit* a, + int n); +extern WOLFSSL_LOCAL sp_digit sp_521_sub_in_place_9(sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_521_mul_d_9(sp_digit* r, const sp_digit* a, + sp_digit b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_521_mul_d_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit b); +#endif +#ifdef _WIN64 +extern WOLFSSL_LOCAL sp_digit div_521_word_asm_9(sp_digit d1, sp_digit d0, + sp_digit div); +#endif +extern WOLFSSL_LOCAL void sp_521_rshift1_9(sp_digit* r, const sp_digit* a); +extern WOLFSSL_LOCAL void sp_521_div2_mod_9(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL int sp_521_num_bits_9(const sp_digit * a); +#endif +#ifdef WOLFSSL_SP_1024 +extern WOLFSSL_LOCAL void sp_1024_mul_16(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_1024_sqr_16(sp_digit* r, const sp_digit* a); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_1024_mul_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_1024_sqr_avx2_16(sp_digit* r, const sp_digit* a); +#endif +extern WOLFSSL_LOCAL sp_digit sp_1024_add_16(sp_digit* r, const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_1024_sub_in_place_16(sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL sp_digit sp_1024_cond_sub_16(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL sp_digit sp_1024_cond_sub_avx2_16(sp_digit* r, + const sp_digit* a, const sp_digit* b, sp_digit m); +#endif +extern WOLFSSL_LOCAL void sp_1024_mul_d_16(sp_digit* r, const sp_digit* a, + sp_digit b); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_1024_mul_d_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit b); +#endif +#ifdef _WIN64 +extern WOLFSSL_LOCAL sp_digit div_1024_word_asm_16(sp_digit d1, sp_digit d0, + sp_digit div); +#endif +extern WOLFSSL_LOCAL sp_int64 sp_1024_cmp_16(const sp_digit* a, + const sp_digit* b); +extern WOLFSSL_LOCAL void sp_1024_cond_copy_16(sp_digit* r, const sp_digit* a, + sp_digit m); +extern WOLFSSL_LOCAL void sp_1024_mont_reduce_16(sp_digit* a, const sp_digit* m, + sp_digit mp); +extern WOLFSSL_LOCAL void sp_1024_mont_add_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_1024_mont_dbl_16(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL void sp_1024_mont_tpl_16(sp_digit* r, const sp_digit* a, + const sp_digit* m); +extern WOLFSSL_LOCAL void sp_1024_mont_sub_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_1024_mont_div2_16(sp_digit* r, const sp_digit* a, + const sp_digit* m); +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void sp_1024_mont_reduce_avx2_16(sp_digit* a, + const sp_digit* m, sp_digit mp); +extern WOLFSSL_LOCAL void sp_1024_mont_add_avx2_16(sp_digit* r, + const sp_digit* a, const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_1024_mont_dbl_avx2_16(sp_digit* r, + const sp_digit* a, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_1024_mont_tpl_avx2_16(sp_digit* r, + const sp_digit* a, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_1024_mont_sub_avx2_16(sp_digit* r, + const sp_digit* a, const sp_digit* b, const sp_digit* m); +extern WOLFSSL_LOCAL void sp_1024_mont_div2_avx2_16(sp_digit* r, + const sp_digit* a, const sp_digit* m); +#endif +extern WOLFSSL_LOCAL void sp_1024_from_bin_bswap(sp_digit* r, int size, + const byte* a, int n); +#ifndef NO_MOVBE_SUPPORT +extern WOLFSSL_LOCAL void sp_1024_from_bin_movbe(sp_digit* r, int size, + const byte* a, int n); + +#endif +#endif +#endif +#ifdef WOLFSSL_SP_X86_64_ASM +#ifndef WOLFSSL_SP_NO_2048 +#ifndef WOLFSSL_SP_NO_2048 +/* Read big endian unsigned byte array into r. + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WOLFSSL_LOCAL void sp_2048_from_bin_bswap(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, r11, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x100); + r11 = (word64)(0); + goto L_2048_from_bin_bswap_64_end; +L_2048_from_bin_bswap_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_L64(r9, 56)); + r8 = (word64)(WC_L64(r9, 48)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_L64(r9, 40)); + r8 = (word64)(WC_L64(r9, 32)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(r9, 24)); + r8 = (word64)(WC_L64(r9, 16)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_L64(r9, 8)); + r8 = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_2048_from_bin_bswap_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_2048_from_bin_bswap_64_start; + } + goto L_2048_from_bin_bswap_8_end; +L_2048_from_bin_bswap_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_2048_from_bin_bswap_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_2048_from_bin_bswap_8_start; + } + if ((rcx) == (r11)) { + goto L_2048_from_bin_bswap_hi_end; + } + r8 = (word64)(r11); + rax = (word64)(r11); +L_2048_from_bin_bswap_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_2048_from_bin_bswap_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_2048_from_bin_bswap_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_2048_from_bin_bswap_zero_end; + } +L_2048_from_bin_bswap_zero_start: + WC_S64(rdi, 0) = (word64)(r11); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_2048_from_bin_bswap_zero_start; + } +L_2048_from_bin_bswap_zero_end: + ; + (void)rsi; +} + +#ifndef NO_MOVBE_SUPPORT +/* Read big endian unsigned byte array into r. + * Uses the movbe instruction which is an optional instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_2048_from_bin_movbe(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x100); + goto L_2048_from_bin_movbe_64_end; +L_2048_from_bin_movbe_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 56))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 48))); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 40))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 32))); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 24))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 16))); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 8))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_2048_from_bin_movbe_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_2048_from_bin_movbe_64_start; + } + goto L_2048_from_bin_movbe_8_end; +L_2048_from_bin_movbe_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_2048_from_bin_movbe_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_2048_from_bin_movbe_8_start; + } + if ((rcx) == (0)) { + goto L_2048_from_bin_movbe_hi_end; + } + r8 = (word64)(0); + rax = (word64)(0); +L_2048_from_bin_movbe_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_2048_from_bin_movbe_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_2048_from_bin_movbe_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_2048_from_bin_movbe_zero_end; + } +L_2048_from_bin_movbe_zero_start: + WC_S64(rdi, 0) = (word64)(0); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_2048_from_bin_movbe_zero_start; + } +L_2048_from_bin_movbe_zero_end: + ; + (void)rsi; +} + +#endif /* !NO_MOVBE_SUPPORT */ +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 256 + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WOLFSSL_LOCAL void sp_2048_to_bin_bswap_32(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rdi, 248)); + rax = (word64)(WC_L64(rdi, 240)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 232)); + rax = (word64)(WC_L64(rdi, 224)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 216)); + rax = (word64)(WC_L64(rdi, 208)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 32) = (word64)(rdx); + WC_S64(rsi, 40) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 200)); + rax = (word64)(WC_L64(rdi, 192)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 48) = (word64)(rdx); + WC_S64(rsi, 56) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 184)); + rax = (word64)(WC_L64(rdi, 176)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 64) = (word64)(rdx); + WC_S64(rsi, 72) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 168)); + rax = (word64)(WC_L64(rdi, 160)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 80) = (word64)(rdx); + WC_S64(rsi, 88) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 152)); + rax = (word64)(WC_L64(rdi, 144)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 96) = (word64)(rdx); + WC_S64(rsi, 104) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 136)); + rax = (word64)(WC_L64(rdi, 128)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 112) = (word64)(rdx); + WC_S64(rsi, 120) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 120)); + rax = (word64)(WC_L64(rdi, 112)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 128) = (word64)(rdx); + WC_S64(rsi, 136) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 104)); + rax = (word64)(WC_L64(rdi, 96)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 144) = (word64)(rdx); + WC_S64(rsi, 152) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 88)); + rax = (word64)(WC_L64(rdi, 80)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 160) = (word64)(rdx); + WC_S64(rsi, 168) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 72)); + rax = (word64)(WC_L64(rdi, 64)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 176) = (word64)(rdx); + WC_S64(rsi, 184) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 56)); + rax = (word64)(WC_L64(rdi, 48)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 192) = (word64)(rdx); + WC_S64(rsi, 200) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 40)); + rax = (word64)(WC_L64(rdi, 32)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 208) = (word64)(rdx); + WC_S64(rsi, 216) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 24)); + rax = (word64)(WC_L64(rdi, 16)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 224) = (word64)(rdx); + WC_S64(rsi, 232) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 8)); + rax = (word64)(WC_L64(rdi, 0)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 240) = (word64)(rdx); + WC_S64(rsi, 248) = (word64)(rax); +} + +#ifndef NO_MOVBE_SUPPORT +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 256 + * Uses the movbe instruction which is optional. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_2048_to_bin_movbe_32(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 248))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 240))); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 232))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 224))); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 216))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 208))); + WC_S64(rsi, 32) = (word64)(rdx); + WC_S64(rsi, 40) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 200))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 192))); + WC_S64(rsi, 48) = (word64)(rdx); + WC_S64(rsi, 56) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 184))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 176))); + WC_S64(rsi, 64) = (word64)(rdx); + WC_S64(rsi, 72) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 168))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 160))); + WC_S64(rsi, 80) = (word64)(rdx); + WC_S64(rsi, 88) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 152))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 144))); + WC_S64(rsi, 96) = (word64)(rdx); + WC_S64(rsi, 104) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 136))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 128))); + WC_S64(rsi, 112) = (word64)(rdx); + WC_S64(rsi, 120) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 120))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 112))); + WC_S64(rsi, 128) = (word64)(rdx); + WC_S64(rsi, 136) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 104))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 96))); + WC_S64(rsi, 144) = (word64)(rdx); + WC_S64(rsi, 152) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 88))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 80))); + WC_S64(rsi, 160) = (word64)(rdx); + WC_S64(rsi, 168) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 72))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 64))); + WC_S64(rsi, 176) = (word64)(rdx); + WC_S64(rsi, 184) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 56))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 48))); + WC_S64(rsi, 192) = (word64)(rdx); + WC_S64(rsi, 200) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 40))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 32))); + WC_S64(rsi, 208) = (word64)(rdx); + WC_S64(rsi, 216) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 24))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 16))); + WC_S64(rsi, 224) = (word64)(rdx); + WC_S64(rsi, 232) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 8))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 0))); + WC_S64(rsi, 240) = (word64)(rdx); + WC_S64(rsi, 248) = (word64)(rax); +} + +#endif /* NO_MOVBE_SUPPORT */ +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_2048_mul_16(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rcx, rsp, rax, rdx = 0, r10, r9, r8; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 8) = (word64)(r9); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 16) = (word64)(r10); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 24) = (word64)(r8); + /* A[0] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 32) = (word64)(r9); + /* A[0] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 40) = (word64)(r10); + /* A[0] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 48) = (word64)(r8); + /* A[0] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 56) = (word64)(r9); + /* A[0] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 64) = (word64)(r10); + /* A[0] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 72) = (word64)(r8); + /* A[0] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 80) = (word64)(r9); + /* A[0] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 88) = (word64)(r10); + /* A[0] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 96) = (word64)(r8); + /* A[0] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 104) = (word64)(r9); + /* A[0] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 112) = (word64)(r10); + /* A[0] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 120) = (word64)(r8); + /* A[1] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 128) = (word64)(r9); + /* A[2] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 136) = (word64)(r10); + /* A[3] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 144) = (word64)(r8); + /* A[4] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 152) = (word64)(r9); + /* A[5] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 160) = (word64)(r10); + /* A[6] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 168) = (word64)(r8); + /* A[7] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 176) = (word64)(r9); + /* A[8] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 184) = (word64)(r10); + /* A[9] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 192) = (word64)(r8); + /* A[10] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 200) = (word64)(r9); + /* A[11] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 208) = (word64)(r10); + /* A[12] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 216) = (word64)(r8); + /* A[13] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 224) = (word64)(r9); + /* A[14] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 232) = (word64)(r10); + /* A[15] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 240) = (word64)(r8); + WC_S64(rdi, 248) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r8 = (word64)(WC_L64(rsp, 16)); + r9 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + r8 = (word64)(WC_L64(rsp, 48)); + r9 = (word64)(WC_L64(rsp, 56)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 64)); + rdx = (word64)(WC_L64(rsp, 72)); + r8 = (word64)(WC_L64(rsp, 80)); + r9 = (word64)(WC_L64(rsp, 88)); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rdx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 96)); + rdx = (word64)(WC_L64(rsp, 104)); + r8 = (word64)(WC_L64(rsp, 112)); + r9 = (word64)(WC_L64(rsp, 120)); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rdx); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply. + * @param [in] b Second number to multiply. + */ +WOLFSSL_LOCAL void sp_2048_mul_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + sp_2048_mul_16(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_2048_add_16(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax, r8; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Add */ + rcx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(0); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 96)); + WC_S64(rdi, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 96), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 104)); + WC_S64(rdi, 96) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 112)); + WC_S64(rdi, 104) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 112), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 120)); + WC_S64(rdi, 112) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 120), (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Sub b from a into a. (a -= b) + * + * @param [in, out] a A single precision integer and result. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_2048_sub_in_place_32(sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rdi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 16), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 24), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 32), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 48), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 56), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 64), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 80), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 88), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 96), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 112), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 120), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 128), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 136), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 144), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 152), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 160), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 168), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 176), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 184), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 192)); + WC_S64(rdi, 184) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 192), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 200)); + WC_S64(rdi, 192) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 200), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 208)); + WC_S64(rdi, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 208), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 216)); + WC_S64(rdi, 208) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 216), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 224)); + WC_S64(rdi, 216) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 224), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 232)); + WC_S64(rdi, 224) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 232), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 240)); + WC_S64(rdi, 232) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 240), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 248)); + WC_S64(rdi, 240) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 248), (unsigned long long*)&rcx); + WC_S64(rdi, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_2048_add_32(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax, r8; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Add */ + rcx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(0); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 96)); + WC_S64(rdi, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 96), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 104)); + WC_S64(rdi, 96) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 112)); + WC_S64(rdi, 104) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 112), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 120)); + WC_S64(rdi, 112) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 120), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(rdi, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 136), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 144)); + WC_S64(rdi, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 144), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 152)); + WC_S64(rdi, 144) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 152), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 160)); + WC_S64(rdi, 152) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 160), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 168)); + WC_S64(rdi, 160) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 168), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(rdi, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 184), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 192)); + WC_S64(rdi, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 192), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 200)); + WC_S64(rdi, 192) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 200), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 208)); + WC_S64(rdi, 200) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 208), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 216)); + WC_S64(rdi, 208) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 216), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 224)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 232)); + WC_S64(rdi, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 232), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 240)); + WC_S64(rdi, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 240), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 248)); + WC_S64(rdi, 240) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 248), (unsigned long long*)&r8); + WC_S64(rdi, 248) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_2048_mul_32(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rsp, r10, r12, rax, r13, rcx, r8, r11, r14, r15, r9; + XALIGNED(32) WC_X64I_SLOT stk[104]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 832; + rsp = (word64)(rsp - 808); + WC_S64(rsp, 768) = (word64)(rdi); + WC_S64(rsp, 776) = (word64)(rsi); + WC_S64(rsp, 784) = (word64)(rdx); + r10 = (word64)(rsp + 512); + r12 = (word64)(rsi + 128); + /* Add */ + rax = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + WC_S64(r10, 120) = (word64)(rax); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 792) = (word64)(r13); + r11 = (word64)(rsp + 640); + r12 = (word64)(rdx + 128); + /* Add */ + rax = (word64)(WC_L64(rdx, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 8)); + WC_S64(r11, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 16)); + WC_S64(r11, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 24)); + WC_S64(r11, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 32)); + WC_S64(r11, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 40)); + WC_S64(r11, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 48)); + WC_S64(r11, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 56)); + WC_S64(r11, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 64)); + WC_S64(r11, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 72)); + WC_S64(r11, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 80)); + WC_S64(r11, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 88)); + WC_S64(r11, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 96)); + WC_S64(r11, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 104)); + WC_S64(r11, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 112)); + WC_S64(r11, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 120)); + WC_S64(r11, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + WC_S64(r11, 120) = (word64)(rax); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 800) = (word64)(r14); + rdx = (word64)(r11); + rsi = (word64)(r10); + rdi = (word64)(rsp); + (void)sp_2048_mul_16((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 784)); + rsi = (word64)(WC_L64(rsp, 776)); + rdi = (word64)(rsp + 256); + rdx = (word64)(rdx + 0x80); + rsi = (word64)(rsi + 0x80); + (void)sp_2048_mul_16((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 784)); + rsi = (word64)(WC_L64(rsp, 776)); + rdi = (word64)(WC_L64(rsp, 768)); + (void)sp_2048_mul_16((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); +#ifdef _WIN64 + rdx = (word64)(WC_L64(rsp, 784)); + rsi = (word64)(WC_L64(rsp, 776)); + rdi = (word64)(WC_L64(rsp, 768)); +#endif /* _WIN64 */ + r13 = (word64)(WC_L64(rsp, 792)); + r14 = (word64)(WC_L64(rsp, 800)); + r15 = (word64)(WC_L64(rsp, 768)); + r9 = (word64)(r13); + r10 = (word64)(rsp + 512); + r11 = (word64)(rsp + 640); + r9 = (word64)(r9 & r14); + r13 = (word64)(0 - r13); + r14 = (word64)(0 - r14); + r15 = (word64)(r15 + 0x100); + rax = (word64)(WC_L64(r10, 0)); + rcx = (word64)(WC_L64(r11, 0)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 0) = (word64)(rax); + WC_S64(r11, 0) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 8)); + rcx = (word64)(WC_L64(r11, 8)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 8) = (word64)(rax); + WC_S64(r11, 8) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 16)); + rcx = (word64)(WC_L64(r11, 16)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 16) = (word64)(rax); + WC_S64(r11, 16) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 24)); + rcx = (word64)(WC_L64(r11, 24)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 24) = (word64)(rax); + WC_S64(r11, 24) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 32)); + rcx = (word64)(WC_L64(r11, 32)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 32) = (word64)(rax); + WC_S64(r11, 32) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 40)); + rcx = (word64)(WC_L64(r11, 40)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 40) = (word64)(rax); + WC_S64(r11, 40) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 48)); + rcx = (word64)(WC_L64(r11, 48)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 48) = (word64)(rax); + WC_S64(r11, 48) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 56)); + rcx = (word64)(WC_L64(r11, 56)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 56) = (word64)(rax); + WC_S64(r11, 56) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 64)); + rcx = (word64)(WC_L64(r11, 64)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 64) = (word64)(rax); + WC_S64(r11, 64) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 72)); + rcx = (word64)(WC_L64(r11, 72)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 72) = (word64)(rax); + WC_S64(r11, 72) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 80)); + rcx = (word64)(WC_L64(r11, 80)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 80) = (word64)(rax); + WC_S64(r11, 80) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 88)); + rcx = (word64)(WC_L64(r11, 88)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 88) = (word64)(rax); + WC_S64(r11, 88) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 96)); + rcx = (word64)(WC_L64(r11, 96)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 96) = (word64)(rax); + WC_S64(r11, 96) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 104)); + rcx = (word64)(WC_L64(r11, 104)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 104) = (word64)(rax); + WC_S64(r11, 104) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 112)); + rcx = (word64)(WC_L64(r11, 112)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 112) = (word64)(rax); + WC_S64(r11, 112) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 120)); + rcx = (word64)(WC_L64(r11, 120)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 120) = (word64)(rax); + WC_S64(r11, 120) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r11 = (word64)(rsp + 256); + r10 = (word64)(rsp); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 248), (unsigned long long*)&rcx); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(rdi, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 248), (unsigned long long*)&rcx); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + r15 = (word64)(r15 - 0x80); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r10, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 200)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 208)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 216)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 224)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 232)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 240)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 248)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 248), (unsigned long long*)&rcx); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 384) = (word64)(r9); + r15 = (word64)(r15 + 0x80); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + WC_S64(r15, 128) = (word64)(rcx); + /* Add to zero */ + rax = (word64)(WC_L64(r11, 136)); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 144)); + WC_S64(r15, 136) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 152)); + WC_S64(r15, 144) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 160)); + WC_S64(r15, 152) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 168)); + WC_S64(r15, 160) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 176)); + WC_S64(r15, 168) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 184)); + WC_S64(r15, 176) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 192)); + WC_S64(r15, 184) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 200)); + WC_S64(r15, 192) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 208)); + WC_S64(r15, 200) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 216)); + WC_S64(r15, 208) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 224)); + WC_S64(r15, 216) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 232)); + WC_S64(r15, 224) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 240)); + WC_S64(r15, 232) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 248)); + WC_S64(r15, 240) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(r15, 248) = (word64)(r8); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_2048_mul_avx2_32(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rsp, r10, r12, rax, r13, rcx, r8, r11, r14, r15, r9; + XALIGNED(32) WC_X64I_SLOT stk[104]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 832; + rsp = (word64)(rsp - 808); + WC_S64(rsp, 768) = (word64)(rdi); + WC_S64(rsp, 776) = (word64)(rsi); + WC_S64(rsp, 784) = (word64)(rdx); + r10 = (word64)(rsp + 512); + r12 = (word64)(rsi + 128); + /* Add */ + rax = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + WC_S64(r10, 120) = (word64)(rax); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 792) = (word64)(r13); + r11 = (word64)(rsp + 640); + r12 = (word64)(rdx + 128); + /* Add */ + rax = (word64)(WC_L64(rdx, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 8)); + WC_S64(r11, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 16)); + WC_S64(r11, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 24)); + WC_S64(r11, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 32)); + WC_S64(r11, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 40)); + WC_S64(r11, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 48)); + WC_S64(r11, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 56)); + WC_S64(r11, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 64)); + WC_S64(r11, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 72)); + WC_S64(r11, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 80)); + WC_S64(r11, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 88)); + WC_S64(r11, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 96)); + WC_S64(r11, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 104)); + WC_S64(r11, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 112)); + WC_S64(r11, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 120)); + WC_S64(r11, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + WC_S64(r11, 120) = (word64)(rax); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 800) = (word64)(r14); + rdx = (word64)(r11); + rsi = (word64)(r10); + rdi = (word64)(rsp); + (void)sp_2048_mul_avx2_16((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 784)); + rsi = (word64)(WC_L64(rsp, 776)); + rdi = (word64)(rsp + 256); + rdx = (word64)(rdx + 0x80); + rsi = (word64)(rsi + 0x80); + (void)sp_2048_mul_avx2_16((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 784)); + rsi = (word64)(WC_L64(rsp, 776)); + rdi = (word64)(WC_L64(rsp, 768)); + (void)sp_2048_mul_avx2_16((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); +#ifdef _WIN64 + rdx = (word64)(WC_L64(rsp, 784)); + rsi = (word64)(WC_L64(rsp, 776)); + rdi = (word64)(WC_L64(rsp, 768)); +#endif /* _WIN64 */ + r13 = (word64)(WC_L64(rsp, 792)); + r14 = (word64)(WC_L64(rsp, 800)); + r15 = (word64)(WC_L64(rsp, 768)); + r9 = (word64)(r13); + r10 = (word64)(rsp + 512); + r11 = (word64)(rsp + 640); + r9 = (word64)(r9 & r14); + r13 = (word64)(0 - r13); + r14 = (word64)(0 - r14); + r15 = (word64)(r15 + 0x100); + rax = (word64)(WC_L64(r10, 0)); + rcx = (word64)(WC_L64(r11, 0)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + r8 = (word64)(WC_L64(r11, 8)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + rax = (word64)(WC_L64(r11, 16)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + rcx = (word64)(WC_L64(r11, 24)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + r8 = (word64)(WC_L64(r11, 32)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + rax = (word64)(WC_L64(r11, 40)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + rcx = (word64)(WC_L64(r11, 48)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + r8 = (word64)(WC_L64(r11, 56)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + rax = (word64)(WC_L64(r11, 64)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + rcx = (word64)(WC_L64(r11, 72)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + r8 = (word64)(WC_L64(r11, 80)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + rax = (word64)(WC_L64(r11, 88)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + rcx = (word64)(WC_L64(r11, 96)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + r8 = (word64)(WC_L64(r11, 104)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + rax = (word64)(WC_L64(r11, 112)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + rcx = (word64)(WC_L64(r11, 120)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r11 = (word64)(rsp + 256); + r10 = (word64)(rsp); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 248), (unsigned long long*)&rcx); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(rdi, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 248), (unsigned long long*)&rcx); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + r15 = (word64)(r15 - 0x80); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r10, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 200)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 208)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 216)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 224)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 232)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 240)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 248)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 248), (unsigned long long*)&rcx); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 384) = (word64)(r9); + r15 = (word64)(r15 + 0x80); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + WC_S64(r15, 128) = (word64)(rcx); + /* Add to zero */ + rax = (word64)(WC_L64(r11, 136)); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 144)); + WC_S64(r15, 136) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 152)); + WC_S64(r15, 144) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 160)); + WC_S64(r15, 152) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 168)); + WC_S64(r15, 160) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 176)); + WC_S64(r15, 168) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 184)); + WC_S64(r15, 176) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 192)); + WC_S64(r15, 184) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 200)); + WC_S64(r15, 192) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 208)); + WC_S64(r15, 200) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 216)); + WC_S64(r15, 208) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 224)); + WC_S64(r15, 216) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 232)); + WC_S64(r15, 224) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 240)); + WC_S64(r15, 232) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 248)); + WC_S64(r15, 240) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(r15, 248) = (word64)(r8); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_2048_sqr_16(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rax, rdx = 0, r9, r8, rcx, r12, r10, r11; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 8) = (word64)(r8); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 16) = (word64)(r9); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 24) = (word64)(rcx); + /* A[0] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 32) = (word64)(r8); + /* A[0] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 40) = (word64)(r9); + /* A[0] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 48) = (word64)(rcx); + /* A[0] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rsp, 56) = (word64)(r8); + /* A[0] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 64) = (word64)(r9); + /* A[0] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 72) = (word64)(rcx); + /* A[0] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rsp, 80) = (word64)(r8); + /* A[0] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 88) = (word64)(r9); + /* A[0] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 96) = (word64)(rcx); + /* A[0] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rsp, 104) = (word64)(r8); + /* A[0] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 112) = (word64)(r9); + /* A[0] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 120) = (word64)(rcx); + /* A[1] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[2] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 128) = (word64)(r8); + /* A[2] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[3] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rdi, 136) = (word64)(r9); + /* A[3] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[4] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rdi, 144) = (word64)(rcx); + /* A[4] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[5] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 152) = (word64)(r8); + /* A[5] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[6] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[10] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rdi, 160) = (word64)(r9); + /* A[6] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[7] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[10] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rdi, 168) = (word64)(rcx); + /* A[7] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[8] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[10] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[11] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 176) = (word64)(r8); + /* A[8] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[9] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[10] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[11] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rdi, 184) = (word64)(r9); + /* A[9] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[10] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[11] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[12] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rdi, 192) = (word64)(rcx); + /* A[10] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[11] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[12] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 200) = (word64)(r8); + /* A[11] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 208) = (word64)(r9); + /* A[12] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 216) = (word64)(rcx); + /* A[13] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[14] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 224) = (word64)(r8); + /* A[14] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 232) = (word64)(r9); + /* A[15] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 240) = (word64)(rcx); + WC_S64(rdi, 248) = (word64)(r8); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r10 = (word64)(WC_L64(rsp, 16)); + r11 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + r10 = (word64)(WC_L64(rsp, 48)); + r11 = (word64)(WC_L64(rsp, 56)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(r10); + WC_S64(rdi, 56) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 64)); + rdx = (word64)(WC_L64(rsp, 72)); + r10 = (word64)(WC_L64(rsp, 80)); + r11 = (word64)(WC_L64(rsp, 88)); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rdx); + WC_S64(rdi, 80) = (word64)(r10); + WC_S64(rdi, 88) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 96)); + rdx = (word64)(WC_L64(rsp, 104)); + r10 = (word64)(WC_L64(rsp, 112)); + r11 = (word64)(WC_L64(rsp, 120)); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rdx); + WC_S64(rdi, 112) = (word64)(r10); + WC_S64(rdi, 120) = (word64)(r11); +} + +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_2048_sqr_avx2_16(sp_digit* r, const sp_digit* a) +{ + sp_2048_sqr_16(r, a); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * Karatsuba: ah^2, al^2, (al - ah)^2 + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_2048_sqr_32(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rcx, r8, r9, rdx, rax; + XALIGNED(32) WC_X64I_SLOT stk[36]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 288; + rsp = (word64)(rsp - 272); + WC_S64(rsp, 256) = (word64)(rdi); + WC_S64(rsp, 264) = (word64)(rsi); + rcx = (word64)(0); + r8 = (word64)(rsp); + r9 = (word64)(rsi + 128); + rdx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(r9, 0), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 8)); + WC_S64(r8, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 8), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 16)); + WC_S64(r8, 8) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 16), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r8, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 24), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 32)); + WC_S64(r8, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 32), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 40)); + WC_S64(r8, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 40), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 48)); + WC_S64(r8, 40) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 48), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 56)); + WC_S64(r8, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 56), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 64)); + WC_S64(r8, 56) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 64), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r8, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 72), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 80)); + WC_S64(r8, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 80), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 88)); + WC_S64(r8, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 88), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 96)); + WC_S64(r8, 88) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 96), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 104)); + WC_S64(r8, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 104), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 112)); + WC_S64(r8, 104) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 112), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r8, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 120), (unsigned long long*)&rax); + WC_S64(r8, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* Cond Negate */ + rdx = (word64)(WC_L64(r8, 0)); + r9 = (word64)(rcx); + rdx = (word64)(rdx ^ rcx); + r9 = (word64)(0 - r9); + cf = _subborrow_u64(0, rdx, rcx, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 8)); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(r8, 0) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 16)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 8) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 24)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 16) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 32)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 24) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 40)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 32) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 48)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 40) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 56)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 48) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 64)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 56) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 72)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 64) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 80)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 72) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 88)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 80) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 96)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 88) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 104)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 96) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 112)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 104) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 120)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 112) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + rax = (word64)(rax + r9); + WC_S64(r8, 120) = (word64)(rax); + rsi = (word64)(r8); + rdi = (word64)(rsp); + (void)sp_2048_sqr_16((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); + rsi = (word64)(WC_L64(rsp, 264)); + rdi = (word64)(WC_L64(rsp, 256)); + rsi = (word64)(rsi + 0x80); + rdi = (word64)(rdi + 0x100); + (void)sp_2048_sqr_16((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); + rsi = (word64)(WC_L64(rsp, 264)); + rdi = (word64)(WC_L64(rsp, 256)); + (void)sp_2048_sqr_16((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); +#ifdef _WIN64 + rsi = (word64)(WC_L64(rsp, 264)); + rdi = (word64)(WC_L64(rsp, 256)); +#endif /* _WIN64 */ + rsi = (word64)(WC_L64(rsp, 256)); + r8 = (word64)(rsp + 128); + rsi = (word64)(rsi + 0x180); + rcx = (word64)(0); + rax = (word64)(WC_L64(r8, -128)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rsi = (word64)(rsi - 0x100); + rax = (word64)(WC_L64(r8, -128)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 256)); + rcx = (word64)(0 - rcx); + rdi = (word64)(rdi + 0x100); + rax = (word64)(WC_L64(rdi, -128)); + cf = _subborrow_u64(0, rax, WC_L64(r8, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -120)); + WC_S64(rdi, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -112)); + WC_S64(rdi, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -104)); + WC_S64(rdi, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -96)); + WC_S64(rdi, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -88)); + WC_S64(rdi, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -80)); + WC_S64(rdi, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -72)); + WC_S64(rdi, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -64)); + WC_S64(rdi, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -56)); + WC_S64(rdi, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -48)); + WC_S64(rdi, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -40)); + WC_S64(rdi, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -32)); + WC_S64(rdi, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -24)); + WC_S64(rdi, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -16)); + WC_S64(rdi, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -8)); + WC_S64(rdi, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 0)); + WC_S64(rdi, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 120), (unsigned long long*)&rdx); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 256)); + rdi = (word64)(rdi + 0x180); + /* Add in word */ + rax = (word64)(WC_L64(rdi, 0)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 120) = (word64)(rdx); + rsi = (word64)(WC_L64(rsp, 264)); + rdi = (word64)(WC_L64(rsp, 256)); +} + +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * Karatsuba: ah^2, al^2, (al - ah)^2 + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_2048_sqr_avx2_32(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rcx, r8, r9, rdx, rax; + XALIGNED(32) WC_X64I_SLOT stk[36]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 288; + rsp = (word64)(rsp - 272); + WC_S64(rsp, 256) = (word64)(rdi); + WC_S64(rsp, 264) = (word64)(rsi); + rcx = (word64)(0); + r8 = (word64)(rsp); + r9 = (word64)(rsi + 128); + rdx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(r9, 0), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 8)); + WC_S64(r8, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 8), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 16)); + WC_S64(r8, 8) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 16), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r8, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 24), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 32)); + WC_S64(r8, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 32), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 40)); + WC_S64(r8, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 40), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 48)); + WC_S64(r8, 40) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 48), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 56)); + WC_S64(r8, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 56), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 64)); + WC_S64(r8, 56) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 64), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r8, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 72), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 80)); + WC_S64(r8, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 80), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 88)); + WC_S64(r8, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 88), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 96)); + WC_S64(r8, 88) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 96), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 104)); + WC_S64(r8, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 104), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 112)); + WC_S64(r8, 104) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 112), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r8, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 120), (unsigned long long*)&rax); + WC_S64(r8, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* Cond Negate */ + rdx = (word64)(WC_L64(r8, 0)); + r9 = (word64)(rcx); + rdx = (word64)(rdx ^ rcx); + r9 = (word64)(0 - r9); + cf = _subborrow_u64(0, rdx, rcx, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 8)); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(r8, 0) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 16)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 8) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 24)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 16) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 32)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 24) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 40)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 32) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 48)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 40) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 56)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 48) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 64)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 56) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 72)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 64) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 80)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 72) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 88)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 80) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 96)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 88) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 104)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 96) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 112)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 104) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 120)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 112) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + rax = (word64)(rax + r9); + WC_S64(r8, 120) = (word64)(rax); + rsi = (word64)(r8); + rdi = (word64)(rsp); + (void)sp_2048_sqr_avx2_16((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); + rsi = (word64)(WC_L64(rsp, 264)); + rdi = (word64)(WC_L64(rsp, 256)); + rsi = (word64)(rsi + 0x80); + rdi = (word64)(rdi + 0x100); + (void)sp_2048_sqr_avx2_16((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); + rsi = (word64)(WC_L64(rsp, 264)); + rdi = (word64)(WC_L64(rsp, 256)); + (void)sp_2048_sqr_avx2_16((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); +#ifdef _WIN64 + rsi = (word64)(WC_L64(rsp, 264)); + rdi = (word64)(WC_L64(rsp, 256)); +#endif /* _WIN64 */ + rsi = (word64)(WC_L64(rsp, 256)); + r8 = (word64)(rsp + 128); + rsi = (word64)(rsi + 0x180); + rcx = (word64)(0); + rax = (word64)(WC_L64(r8, -128)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rsi = (word64)(rsi - 0x100); + rax = (word64)(WC_L64(r8, -128)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 256)); + rcx = (word64)(0 - rcx); + rdi = (word64)(rdi + 0x100); + rax = (word64)(WC_L64(rdi, -128)); + cf = _subborrow_u64(0, rax, WC_L64(r8, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -120)); + WC_S64(rdi, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -112)); + WC_S64(rdi, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -104)); + WC_S64(rdi, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -96)); + WC_S64(rdi, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -88)); + WC_S64(rdi, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -80)); + WC_S64(rdi, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -72)); + WC_S64(rdi, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -64)); + WC_S64(rdi, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -56)); + WC_S64(rdi, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -48)); + WC_S64(rdi, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -40)); + WC_S64(rdi, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -32)); + WC_S64(rdi, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -24)); + WC_S64(rdi, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -16)); + WC_S64(rdi, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -8)); + WC_S64(rdi, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 0)); + WC_S64(rdi, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 120), (unsigned long long*)&rdx); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 256)); + rdi = (word64)(rdi + 0x180); + /* Add in word */ + rax = (word64)(WC_L64(rdi, 0)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 120) = (word64)(rdx); + rsi = (word64)(WC_L64(rsp, 264)); + rdi = (word64)(WC_L64(rsp, 256)); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Sub b from a into a. (a -= b) + * + * @param [in, out] a A single precision integer and result. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_2048_sub_in_place_16(sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rdi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 16), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 24), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 32), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 48), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 56), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 64), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 80), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 88), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 96), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 112), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 120), (unsigned long long*)&rcx); + WC_S64(rdi, 120) = (word64)(rcx); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_2048_mul_d_32(sp_digit* r, const sp_digit* a, sp_digit b) +{ + word64 rdi, rsi, rcx, rax, r10, rdx = 0, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)b; + + /* A[0] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + WC_S64(rdi, 0) = (word64)(r8); + /* A[1] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 88) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 104) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 112) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[16] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 128)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[17] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 136)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 136) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[18] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 144)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 144) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[19] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 152)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 152) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[20] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 160)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 160) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[21] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 168)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[22] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 176)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[23] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 184)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 184) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[24] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 192)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 192) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[25] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 200)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 200) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[26] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 208)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 208) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[27] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 216)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 216) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[28] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 224)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 224) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[29] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 232)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 232) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[30] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 240)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 240) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[31] * B */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 248)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + WC_S64(rdi, 248) = (word64)(r9); + WC_S64(rdi, 256) = (word64)(r10); +} + +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_2048_cond_sub_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, r8, r9, rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _subborrow_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsp, 72)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 80)); + rdx = (word64)(WC_L64(rsp, 80)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsp, 88)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 96)); + rdx = (word64)(WC_L64(rsp, 96)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsp, 104)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 112)); + rdx = (word64)(WC_L64(rsp, 112)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsp, 120)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Reduce the number back to 2048 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_2048_mont_reduce_16(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rsi, rcx, r15, r8, r13, r14, r11 = 0, rax = 0, r10 = 0, + rdx = 0, r9 = 0, r12 = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)mp; + + r15 = (word64)(0); + /* i = 16 */ + r8 = (word64)(0x10); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 8)); +L_2048_mont_reduce_16_loop: + /* mu = a[i] * mp */ + r11 = (word64)(r13); + r11 = (word64)(r11 * rcx); + /* a[i+0] += m[0] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* a[i+1] += m[1] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r13, r10, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+2] += m[2] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+3] += m[3] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 24) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+4] += m[4] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rdi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+5] += m[5] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 40) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+6] += m[6] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rdi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 48) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+7] += m[7] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r12 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 56) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+8] += m[8] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r12 = (word64)(WC_L64(rdi, 64)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 64) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+9] += m[9] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r12 = (word64)(WC_L64(rdi, 72)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 72) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+10] += m[10] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r12 = (word64)(WC_L64(rdi, 80)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 80) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+11] += m[11] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r12 = (word64)(WC_L64(rdi, 88)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 88) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+12] += m[12] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r12 = (word64)(WC_L64(rdi, 96)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 96) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+13] += m[13] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + r12 = (word64)(WC_L64(rdi, 104)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 104) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+14] += m[14] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r12 = (word64)(WC_L64(rdi, 112)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 112) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+15] += m[15] * mu */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + r12 = (word64)(WC_L64(rdi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, r15, (unsigned long long*)&rdx); + r15 = (word64)(0); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 120) = (word64)(r12); + cf = _addcarry_u64(cf, WC_L64(rdi, 128), rdx, (unsigned long long*)&WC_S64( + rdi, 128)); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* i -= 1 */ + rdi = (word64)(rdi + 8); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_2048_mont_reduce_16_loop; + } + WC_S64(rdi, 0) = (word64)(r13); + WC_S64(rdi, 8) = (word64)(r14); + r15 = (word64)(0 - r15); +#ifdef _WIN64 + rdx = (word64)(rsi); + rcx = (word64)(r15); +#else + rcx = (word64)(r15); + rdx = (word64)(rsi); +#endif /* _WIN64 */ + rsi = (word64)(rdi); + rdi = (word64)(rdi); + rdi = (word64)(rdi - 0x80); + (void)sp_2048_cond_sub_16((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx, (sp_digit)rcx); +} + +#ifdef HAVE_INTEL_AVX2 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_2048_cond_sub_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r10, r8, r9, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _subborrow_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 64) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 80) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 88) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 104)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 96) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 104) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 112) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r8); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_2048_mul_d_16(sp_digit* r, const sp_digit* a, sp_digit b) +{ + word64 rdi, rsi, rcx, rax, r10, rdx = 0, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)b; + + /* A[0] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + WC_S64(rdi, 0) = (word64)(r8); + /* A[1] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 88) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 104) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 112) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 120) = (word64)(r8); + WC_S64(rdi, 128) = (word64)(r9); +} + +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_2048_mul_d_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit b) +{ + sp_2048_mul_d_16(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef _WIN64 +/* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) + * + * @param [in] d1 The high order half of the number to divide. + * @param [in] d0 The low order half of the number to divide. + * @param [in] div The dividend. + * + * @return The result of the division. + */ +WOLFSSL_LOCAL sp_digit div_2048_word_asm_16(sp_digit d1, sp_digit d0, + sp_digit div) +{ + word64 rdi, rsi, rcx, rax, rdx; + + rdi = (word64)(word64)d1; + rsi = (word64)(word64)d0; + rcx = (word64)(word64)div; + + rax = (word64)(rsi); + rdx = (word64)(rdi); + WC_X64I_DIV128(rax, rdx, rdx, rax, rcx); + return (sp_digit)(word64)rax; +} + +#endif /* _WIN64 */ +/* Compare a with b in constant time. + * + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + * + * @return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +WOLFSSL_LOCAL sp_int64 sp_2048_cmp_16(const sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + word64 zf7; + word64 zf8; + word64 zf9; + word64 zf10; + word64 zf11; + word64 zf12; + word64 zf13; + word64 zf14; + word64 zf15; + word64 zf16; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rcx = (word64)(0); + rdx = (word64)(-1); + rax = (word64)(-1); + r8 = (word64)(1); + r9 = (word64)(WC_L64(rdi, 120)); + r10 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf1 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf1) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf2 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf2) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 104)); + r10 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf3 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf3) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf4 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf4) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf5 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf5) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf6 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf6) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf7 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf7) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf8 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf8) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(WC_L64(rsi, 56)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf9 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf9) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf10 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf10) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf11 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf11) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf12 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf12) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf13 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf13) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf14 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf14) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf15 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf15) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf16 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf16) != (0) ? rcx : rdx; + rax = (word64)(rax ^ rdx); + return (sp_int64)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 2048 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_2048_mont_reduce_avx2_16(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + sp_2048_mont_reduce_16(a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifndef WC_NO_CACHE_RESISTANT +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_2048_get_from_table_avx2_16(sp_digit* r, sp_digit** table, + int idx) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y10, y11, y12, y13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y10 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rdx)); + y11 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rax)); + y13 = _mm256_setzero_si256(); + y10 = _mm256_permutevar8x32_epi32(y10, y13); + y11 = _mm256_permutevar8x32_epi32(y11, y13); + /* START: 0-15 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 16 */ + rcx = (word64)(WC_L64(rsi, 128)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 17 */ + rcx = (word64)(WC_L64(rsi, 136)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 18 */ + rcx = (word64)(WC_L64(rsi, 144)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 19 */ + rcx = (word64)(WC_L64(rsi, 152)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 20 */ + rcx = (word64)(WC_L64(rsi, 160)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 21 */ + rcx = (word64)(WC_L64(rsi, 168)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 22 */ + rcx = (word64)(WC_L64(rsi, 176)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 23 */ + rcx = (word64)(WC_L64(rsi, 184)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 24 */ + rcx = (word64)(WC_L64(rsi, 192)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 25 */ + rcx = (word64)(WC_L64(rsi, 200)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 26 */ + rcx = (word64)(WC_L64(rsi, 208)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 27 */ + rcx = (word64)(WC_L64(rsi, 216)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 28 */ + rcx = (word64)(WC_L64(rsi, 224)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 29 */ + rcx = (word64)(WC_L64(rsi, 232)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 30 */ + rcx = (word64)(WC_L64(rsi, 240)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 31 */ + rcx = (word64)(WC_L64(rsi, 248)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + /* END: 0-15 */ +} + +#endif /* !WC_NO_CACHE_RESISTANT */ +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_2048_cond_sub_32(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, r8, r9, rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[32]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 256; + rsp = (word64)(rsp - 256); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rdx, 136)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 128) = (word64)(r8); + WC_S64(rsp, 136) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 144)); + r9 = (word64)(WC_L64(rdx, 152)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 144) = (word64)(r8); + WC_S64(rsp, 152) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 160)); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 160) = (word64)(r8); + WC_S64(rsp, 168) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rdx, 184)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 176) = (word64)(r8); + WC_S64(rsp, 184) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 192)); + r9 = (word64)(WC_L64(rdx, 200)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 192) = (word64)(r8); + WC_S64(rsp, 200) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 208)); + r9 = (word64)(WC_L64(rdx, 216)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 208) = (word64)(r8); + WC_S64(rsp, 216) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 224)); + r9 = (word64)(WC_L64(rdx, 232)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 224) = (word64)(r8); + WC_S64(rsp, 232) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 240)); + r9 = (word64)(WC_L64(rdx, 248)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 240) = (word64)(r8); + WC_S64(rsp, 248) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _subborrow_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsp, 72)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 80)); + rdx = (word64)(WC_L64(rsp, 80)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsp, 88)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 96)); + rdx = (word64)(WC_L64(rsp, 96)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsp, 104)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 112)); + rdx = (word64)(WC_L64(rsp, 112)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsp, 120)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 128)); + rdx = (word64)(WC_L64(rsp, 128)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 136)); + rdx = (word64)(WC_L64(rsp, 136)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 144)); + rdx = (word64)(WC_L64(rsp, 144)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 136) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 152)); + rdx = (word64)(WC_L64(rsp, 152)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 144) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 160)); + rdx = (word64)(WC_L64(rsp, 160)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 152) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 168)); + rdx = (word64)(WC_L64(rsp, 168)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 160) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 176)); + rdx = (word64)(WC_L64(rsp, 176)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 184)); + rdx = (word64)(WC_L64(rsp, 184)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 192)); + rdx = (word64)(WC_L64(rsp, 192)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 184) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 200)); + rdx = (word64)(WC_L64(rsp, 200)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 192) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 208)); + rdx = (word64)(WC_L64(rsp, 208)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 200) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 216)); + rdx = (word64)(WC_L64(rsp, 216)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 208) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 224)); + rdx = (word64)(WC_L64(rsp, 224)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 216) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 232)); + rdx = (word64)(WC_L64(rsp, 232)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 224) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 240)); + rdx = (word64)(WC_L64(rsp, 240)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 232) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 248)); + rdx = (word64)(WC_L64(rsp, 248)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 240) = (word64)(r8); + WC_S64(rdi, 248) = (word64)(r9); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Reduce the number back to 2048 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_2048_mont_reduce_32(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rsi, rcx, r15, r8, r13, r14, r11 = 0, rax = 0, r10 = 0, + rdx = 0, r9 = 0, r12 = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)mp; + + r15 = (word64)(0); + /* i = 32 */ + r8 = (word64)(0x20); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 8)); +L_2048_mont_reduce_32_loop: + /* mu = a[i] * mp */ + r11 = (word64)(r13); + r11 = (word64)(r11 * rcx); + /* a[i+0] += m[0] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* a[i+1] += m[1] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r13, r10, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+2] += m[2] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+3] += m[3] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 24) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+4] += m[4] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rdi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+5] += m[5] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 40) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+6] += m[6] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rdi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 48) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+7] += m[7] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r12 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 56) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+8] += m[8] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r12 = (word64)(WC_L64(rdi, 64)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 64) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+9] += m[9] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r12 = (word64)(WC_L64(rdi, 72)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 72) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+10] += m[10] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r12 = (word64)(WC_L64(rdi, 80)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 80) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+11] += m[11] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r12 = (word64)(WC_L64(rdi, 88)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 88) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+12] += m[12] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r12 = (word64)(WC_L64(rdi, 96)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 96) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+13] += m[13] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + r12 = (word64)(WC_L64(rdi, 104)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 104) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+14] += m[14] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r12 = (word64)(WC_L64(rdi, 112)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 112) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+15] += m[15] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + r12 = (word64)(WC_L64(rdi, 120)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 120) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+16] += m[16] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 128)); + r12 = (word64)(WC_L64(rdi, 128)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 128) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+17] += m[17] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 136)); + r12 = (word64)(WC_L64(rdi, 136)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 136) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+18] += m[18] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 144)); + r12 = (word64)(WC_L64(rdi, 144)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 144) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+19] += m[19] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 152)); + r12 = (word64)(WC_L64(rdi, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 152) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+20] += m[20] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 160)); + r12 = (word64)(WC_L64(rdi, 160)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 160) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+21] += m[21] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 168)); + r12 = (word64)(WC_L64(rdi, 168)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 168) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+22] += m[22] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 176)); + r12 = (word64)(WC_L64(rdi, 176)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 176) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+23] += m[23] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 184)); + r12 = (word64)(WC_L64(rdi, 184)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 184) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+24] += m[24] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 192)); + r12 = (word64)(WC_L64(rdi, 192)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 192) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+25] += m[25] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 200)); + r12 = (word64)(WC_L64(rdi, 200)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 200) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+26] += m[26] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 208)); + r12 = (word64)(WC_L64(rdi, 208)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 208) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+27] += m[27] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 216)); + r12 = (word64)(WC_L64(rdi, 216)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 216) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+28] += m[28] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 224)); + r12 = (word64)(WC_L64(rdi, 224)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 224) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+29] += m[29] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 232)); + r12 = (word64)(WC_L64(rdi, 232)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 232) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+30] += m[30] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 240)); + r12 = (word64)(WC_L64(rdi, 240)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 240) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+31] += m[31] * mu */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 248)); + r12 = (word64)(WC_L64(rdi, 248)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, r15, (unsigned long long*)&rdx); + r15 = (word64)(0); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 248) = (word64)(r12); + cf = _addcarry_u64(cf, WC_L64(rdi, 256), rdx, (unsigned long long*)&WC_S64( + rdi, 256)); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* i -= 1 */ + rdi = (word64)(rdi + 8); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_2048_mont_reduce_32_loop; + } + WC_S64(rdi, 0) = (word64)(r13); + WC_S64(rdi, 8) = (word64)(r14); + r15 = (word64)(0 - r15); +#ifdef _WIN64 + rdx = (word64)(rsi); + rcx = (word64)(r15); +#else + rcx = (word64)(r15); + rdx = (word64)(rsi); +#endif /* _WIN64 */ + rsi = (word64)(rdi); + rdi = (word64)(rdi); + rdi = (word64)(rdi - 0x100); + (void)sp_2048_cond_sub_32((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx, (sp_digit)rcx); +} + +/* Sub b from a into r. (r = a - b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_2048_sub_32(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, r8, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rcx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 96)); + WC_S64(rdi, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 96), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 104)); + WC_S64(rdi, 96) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 112)); + WC_S64(rdi, 104) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 112), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 120)); + WC_S64(rdi, 112) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 120), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(rdi, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 136), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 144)); + WC_S64(rdi, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 144), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 152)); + WC_S64(rdi, 144) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 152), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 160)); + WC_S64(rdi, 152) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 160), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 168)); + WC_S64(rdi, 160) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 168), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(rdi, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 184), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 192)); + WC_S64(rdi, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 192), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 200)); + WC_S64(rdi, 192) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 200), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 208)); + WC_S64(rdi, 200) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 208), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 216)); + WC_S64(rdi, 208) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 216), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 224)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 232)); + WC_S64(rdi, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 232), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 240)); + WC_S64(rdi, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 240), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 248)); + WC_S64(rdi, 240) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 248), (unsigned long long*)&r8); + WC_S64(rdi, 248) = (word64)(r8); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_2048_mul_d_avx2_32(sp_digit* r, const sp_digit* a, + const sp_digit b) +{ + sp_2048_mul_d_32(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef _WIN64 +/* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) + * + * @param [in] d1 The high order half of the number to divide. + * @param [in] d0 The low order half of the number to divide. + * @param [in] div The dividend. + * + * @return The result of the division. + */ +WOLFSSL_LOCAL sp_digit div_2048_word_asm_32(sp_digit d1, sp_digit d0, + sp_digit div) +{ + word64 rdi, rsi, rcx, rax, rdx; + + rdi = (word64)(word64)d1; + rsi = (word64)(word64)d0; + rcx = (word64)(word64)div; + + rax = (word64)(rsi); + rdx = (word64)(rdi); + WC_X64I_DIV128(rax, rdx, rdx, rax, rcx); + return (sp_digit)(word64)rax; +} + +#endif /* _WIN64 */ +#ifdef HAVE_INTEL_AVX2 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_2048_cond_sub_avx2_32(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r10, r8, r9, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _subborrow_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 64) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 80) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 88) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 104)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 96) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 104) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 112) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rsi, 128)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 128) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 144)); + r8 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 136) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 152)); + r9 = (word64)(WC_L64(rsi, 152)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 144) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 152) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 160) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rsi, 176)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 176) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 192)); + r8 = (word64)(WC_L64(rsi, 192)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 184) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 200)); + r9 = (word64)(WC_L64(rsi, 200)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 192) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 208)); + r10 = (word64)(WC_L64(rsi, 208)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 200) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 216)); + r8 = (word64)(WC_L64(rsi, 216)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 208) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 224)); + r9 = (word64)(WC_L64(rsi, 224)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 232)); + r10 = (word64)(WC_L64(rsi, 232)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 224) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 240)); + r8 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 232) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 248)); + r9 = (word64)(WC_L64(rsi, 248)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 240) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + WC_S64(rdi, 248) = (word64)(r9); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Compare a with b in constant time. + * + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + * + * @return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +WOLFSSL_LOCAL sp_int64 sp_2048_cmp_32(const sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + word64 zf7; + word64 zf8; + word64 zf9; + word64 zf10; + word64 zf11; + word64 zf12; + word64 zf13; + word64 zf14; + word64 zf15; + word64 zf16; + word64 zf17; + word64 zf18; + word64 zf19; + word64 zf20; + word64 zf21; + word64 zf22; + word64 zf23; + word64 zf24; + word64 zf25; + word64 zf26; + word64 zf27; + word64 zf28; + word64 zf29; + word64 zf30; + word64 zf31; + word64 zf32; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rcx = (word64)(0); + rdx = (word64)(-1); + rax = (word64)(-1); + r8 = (word64)(1); + r9 = (word64)(WC_L64(rdi, 248)); + r10 = (word64)(WC_L64(rsi, 248)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf1 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf1) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 240)); + r10 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf2 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf2) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 232)); + r10 = (word64)(WC_L64(rsi, 232)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf3 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf3) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 224)); + r10 = (word64)(WC_L64(rsi, 224)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf4 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf4) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 216)); + r10 = (word64)(WC_L64(rsi, 216)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf5 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf5) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 208)); + r10 = (word64)(WC_L64(rsi, 208)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf6 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf6) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 200)); + r10 = (word64)(WC_L64(rsi, 200)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf7 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf7) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 192)); + r10 = (word64)(WC_L64(rsi, 192)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf8 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf8) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf9 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf9) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 176)); + r10 = (word64)(WC_L64(rsi, 176)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf10 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf10) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 168)); + r10 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf11 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf11) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf12 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf12) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 152)); + r10 = (word64)(WC_L64(rsi, 152)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf13 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf13) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 144)); + r10 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf14 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf14) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf15 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf15) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 128)); + r10 = (word64)(WC_L64(rsi, 128)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf16 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf16) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 120)); + r10 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf17 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf17) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf18 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf18) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 104)); + r10 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf19 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf19) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf20 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf20) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf21 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf21) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf22 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf22) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf23 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf23) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf24 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf24) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(WC_L64(rsi, 56)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf25 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf25) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf26 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf26) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf27 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf27) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf28 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf28) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf29 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf29) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf30 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf30) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf31 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf31) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf32 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf32) != (0) ? rcx : rdx; + rax = (word64)(rax ^ rdx); + return (sp_int64)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 2048 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_2048_mont_reduce_avx2_32(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + sp_2048_mont_reduce_32(a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifndef WC_NO_CACHE_RESISTANT +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_2048_get_from_table_avx2_32(sp_digit* r, sp_digit** table, + int idx) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y10, y11, y12, y13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y10 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rdx)); + y11 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rax)); + y13 = _mm256_setzero_si256(); + y10 = _mm256_permutevar8x32_epi32(y10, y13); + y11 = _mm256_permutevar8x32_epi32(y11, y13); + /* START: 0-15 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 16 */ + rcx = (word64)(WC_L64(rsi, 128)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 17 */ + rcx = (word64)(WC_L64(rsi, 136)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 18 */ + rcx = (word64)(WC_L64(rsi, 144)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 19 */ + rcx = (word64)(WC_L64(rsi, 152)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 20 */ + rcx = (word64)(WC_L64(rsi, 160)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 21 */ + rcx = (word64)(WC_L64(rsi, 168)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 22 */ + rcx = (word64)(WC_L64(rsi, 176)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 23 */ + rcx = (word64)(WC_L64(rsi, 184)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 24 */ + rcx = (word64)(WC_L64(rsi, 192)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 25 */ + rcx = (word64)(WC_L64(rsi, 200)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 26 */ + rcx = (word64)(WC_L64(rsi, 208)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 27 */ + rcx = (word64)(WC_L64(rsi, 216)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 28 */ + rcx = (word64)(WC_L64(rsi, 224)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 29 */ + rcx = (word64)(WC_L64(rsi, 232)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 30 */ + rcx = (word64)(WC_L64(rsi, 240)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 31 */ + rcx = (word64)(WC_L64(rsi, 248)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 32 */ + rcx = (word64)(WC_L64(rsi, 256)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 33 */ + rcx = (word64)(WC_L64(rsi, 264)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 34 */ + rcx = (word64)(WC_L64(rsi, 272)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 35 */ + rcx = (word64)(WC_L64(rsi, 280)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 36 */ + rcx = (word64)(WC_L64(rsi, 288)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 37 */ + rcx = (word64)(WC_L64(rsi, 296)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 38 */ + rcx = (word64)(WC_L64(rsi, 304)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 39 */ + rcx = (word64)(WC_L64(rsi, 312)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 40 */ + rcx = (word64)(WC_L64(rsi, 320)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 41 */ + rcx = (word64)(WC_L64(rsi, 328)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 42 */ + rcx = (word64)(WC_L64(rsi, 336)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 43 */ + rcx = (word64)(WC_L64(rsi, 344)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 44 */ + rcx = (word64)(WC_L64(rsi, 352)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 45 */ + rcx = (word64)(WC_L64(rsi, 360)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 46 */ + rcx = (word64)(WC_L64(rsi, 368)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 47 */ + rcx = (word64)(WC_L64(rsi, 376)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 48 */ + rcx = (word64)(WC_L64(rsi, 384)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 49 */ + rcx = (word64)(WC_L64(rsi, 392)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 50 */ + rcx = (word64)(WC_L64(rsi, 400)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 51 */ + rcx = (word64)(WC_L64(rsi, 408)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 52 */ + rcx = (word64)(WC_L64(rsi, 416)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 53 */ + rcx = (word64)(WC_L64(rsi, 424)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 54 */ + rcx = (word64)(WC_L64(rsi, 432)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 55 */ + rcx = (word64)(WC_L64(rsi, 440)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 56 */ + rcx = (word64)(WC_L64(rsi, 448)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 57 */ + rcx = (word64)(WC_L64(rsi, 456)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 58 */ + rcx = (word64)(WC_L64(rsi, 464)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 59 */ + rcx = (word64)(WC_L64(rsi, 472)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 60 */ + rcx = (word64)(WC_L64(rsi, 480)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 61 */ + rcx = (word64)(WC_L64(rsi, 488)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 62 */ + rcx = (word64)(WC_L64(rsi, 496)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 63 */ + rcx = (word64)(WC_L64(rsi, 504)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + rdi = (word64)(rdi + 0x80); + /* END: 0-15 */ + /* START: 16-31 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 16 */ + rcx = (word64)(WC_L64(rsi, 128)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 17 */ + rcx = (word64)(WC_L64(rsi, 136)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 18 */ + rcx = (word64)(WC_L64(rsi, 144)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 19 */ + rcx = (word64)(WC_L64(rsi, 152)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 20 */ + rcx = (word64)(WC_L64(rsi, 160)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 21 */ + rcx = (word64)(WC_L64(rsi, 168)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 22 */ + rcx = (word64)(WC_L64(rsi, 176)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 23 */ + rcx = (word64)(WC_L64(rsi, 184)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 24 */ + rcx = (word64)(WC_L64(rsi, 192)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 25 */ + rcx = (word64)(WC_L64(rsi, 200)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 26 */ + rcx = (word64)(WC_L64(rsi, 208)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 27 */ + rcx = (word64)(WC_L64(rsi, 216)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 28 */ + rcx = (word64)(WC_L64(rsi, 224)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 29 */ + rcx = (word64)(WC_L64(rsi, 232)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 30 */ + rcx = (word64)(WC_L64(rsi, 240)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 31 */ + rcx = (word64)(WC_L64(rsi, 248)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 32 */ + rcx = (word64)(WC_L64(rsi, 256)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 33 */ + rcx = (word64)(WC_L64(rsi, 264)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 34 */ + rcx = (word64)(WC_L64(rsi, 272)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 35 */ + rcx = (word64)(WC_L64(rsi, 280)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 36 */ + rcx = (word64)(WC_L64(rsi, 288)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 37 */ + rcx = (word64)(WC_L64(rsi, 296)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 38 */ + rcx = (word64)(WC_L64(rsi, 304)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 39 */ + rcx = (word64)(WC_L64(rsi, 312)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 40 */ + rcx = (word64)(WC_L64(rsi, 320)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 41 */ + rcx = (word64)(WC_L64(rsi, 328)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 42 */ + rcx = (word64)(WC_L64(rsi, 336)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 43 */ + rcx = (word64)(WC_L64(rsi, 344)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 44 */ + rcx = (word64)(WC_L64(rsi, 352)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 45 */ + rcx = (word64)(WC_L64(rsi, 360)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 46 */ + rcx = (word64)(WC_L64(rsi, 368)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 47 */ + rcx = (word64)(WC_L64(rsi, 376)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 48 */ + rcx = (word64)(WC_L64(rsi, 384)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 49 */ + rcx = (word64)(WC_L64(rsi, 392)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 50 */ + rcx = (word64)(WC_L64(rsi, 400)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 51 */ + rcx = (word64)(WC_L64(rsi, 408)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 52 */ + rcx = (word64)(WC_L64(rsi, 416)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 53 */ + rcx = (word64)(WC_L64(rsi, 424)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 54 */ + rcx = (word64)(WC_L64(rsi, 432)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 55 */ + rcx = (word64)(WC_L64(rsi, 440)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 56 */ + rcx = (word64)(WC_L64(rsi, 448)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 57 */ + rcx = (word64)(WC_L64(rsi, 456)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 58 */ + rcx = (word64)(WC_L64(rsi, 464)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 59 */ + rcx = (word64)(WC_L64(rsi, 472)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 60 */ + rcx = (word64)(WC_L64(rsi, 480)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 61 */ + rcx = (word64)(WC_L64(rsi, 488)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 62 */ + rcx = (word64)(WC_L64(rsi, 496)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 63 */ + rcx = (word64)(WC_L64(rsi, 504)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + /* END: 16-31 */ +} + +#endif /* !WC_NO_CACHE_RESISTANT */ +/* Conditionally add a and b using the mask m. + * m is -1 to add and 0 when not. + * + * @param [out] r A single precision number representing conditional add + * result. + * @param [in] a A single precision number to add with. + * @param [in] b A single precision number to add. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_2048_cond_add_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, rax, r8, r9; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + rax = (word64)(0); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _addcarry_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsp, 72)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 80)); + rdx = (word64)(WC_L64(rsp, 80)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsp, 88)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 96)); + rdx = (word64)(WC_L64(rsp, 96)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsp, 104)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 112)); + rdx = (word64)(WC_L64(rsp, 112)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsp, 120)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Conditionally add a and b using the mask m. + * m is -1 to add and 0 when not. + * + * @param [out] r A single precision number representing conditional add + * result. + * @param [in] a A single precision number to add with. + * @param [in] b A single precision number to add. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_2048_cond_add_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rax = (word64)(0); + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 64) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 80) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 88) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 104)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 96) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 104) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 112) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Shift number left by n bit. (r = a << n) + * + * @param [out] r Result of left shift by n. + * @param [in] a Number to shift. + * @param [in] n Amoutnt o shift. + */ +WOLFSSL_LOCAL void sp_2048_lshift_32(sp_digit* r, const sp_digit* a, int n) +{ + word64 rdi, rsi, rcx, r10, r11, rdx, rax, r8, r9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r10 = (word64)(0); + r11 = (word64)(WC_L64(rsi, 216)); + rdx = (word64)(WC_L64(rsi, 224)); + rax = (word64)(WC_L64(rsi, 232)); + r8 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)(WC_L64(rsi, 248)); + r10 = (word64)((r10 << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 224) = (word64)(rdx); + WC_S64(rdi, 232) = (word64)(rax); + WC_S64(rdi, 240) = (word64)(r8); + WC_S64(rdi, 248) = (word64)(r9); + WC_S64(rdi, 256) = (word64)(r10); + r9 = (word64)(WC_L64(rsi, 184)); + rdx = (word64)(WC_L64(rsi, 192)); + rax = (word64)(WC_L64(rsi, 200)); + r8 = (word64)(WC_L64(rsi, 208)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 192) = (word64)(rdx); + WC_S64(rdi, 200) = (word64)(rax); + WC_S64(rdi, 208) = (word64)(r8); + WC_S64(rdi, 216) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 152)); + rdx = (word64)(WC_L64(rsi, 160)); + rax = (word64)(WC_L64(rsi, 168)); + r8 = (word64)(WC_L64(rsi, 176)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 160) = (word64)(rdx); + WC_S64(rdi, 168) = (word64)(rax); + WC_S64(rdi, 176) = (word64)(r8); + WC_S64(rdi, 184) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsi, 128)); + rax = (word64)(WC_L64(rsi, 136)); + r8 = (word64)(WC_L64(rsi, 144)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 128) = (word64)(rdx); + WC_S64(rdi, 136) = (word64)(rax); + WC_S64(rdi, 144) = (word64)(r8); + WC_S64(rdi, 152) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsi, 96)); + rax = (word64)(WC_L64(rsi, 104)); + r8 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 96) = (word64)(rdx); + WC_S64(rdi, 104) = (word64)(rax); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsi, 64)); + rax = (word64)(WC_L64(rsi, 72)); + r8 = (word64)(WC_L64(rsi, 80)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 64) = (word64)(rdx); + WC_S64(rdi, 72) = (word64)(rax); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsi, 32)); + rax = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 32) = (word64)(rdx); + WC_S64(rdi, 40) = (word64)(rax); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)(rdx << ((byte)rcx & 63)); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r11); +} + +#endif /* !WOLFSSL_SP_NO_2048 */ +#endif /* !WOLFSSL_SP_NO_2048 */ +#ifndef WOLFSSL_SP_NO_3072 +#ifndef WOLFSSL_SP_NO_3072 +/* Read big endian unsigned byte array into r. + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WOLFSSL_LOCAL void sp_3072_from_bin_bswap(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, r11, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x180); + r11 = (word64)(0); + goto L_3072_from_bin_bswap_64_end; +L_3072_from_bin_bswap_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_L64(r9, 56)); + r8 = (word64)(WC_L64(r9, 48)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_L64(r9, 40)); + r8 = (word64)(WC_L64(r9, 32)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(r9, 24)); + r8 = (word64)(WC_L64(r9, 16)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_L64(r9, 8)); + r8 = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_3072_from_bin_bswap_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_3072_from_bin_bswap_64_start; + } + goto L_3072_from_bin_bswap_8_end; +L_3072_from_bin_bswap_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_3072_from_bin_bswap_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_3072_from_bin_bswap_8_start; + } + if ((rcx) == (r11)) { + goto L_3072_from_bin_bswap_hi_end; + } + r8 = (word64)(r11); + rax = (word64)(r11); +L_3072_from_bin_bswap_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_3072_from_bin_bswap_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_3072_from_bin_bswap_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_3072_from_bin_bswap_zero_end; + } +L_3072_from_bin_bswap_zero_start: + WC_S64(rdi, 0) = (word64)(r11); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_3072_from_bin_bswap_zero_start; + } +L_3072_from_bin_bswap_zero_end: + ; + (void)rsi; +} + +#ifndef NO_MOVBE_SUPPORT +/* Read big endian unsigned byte array into r. + * Uses the movbe instruction which is an optional instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_3072_from_bin_movbe(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x180); + goto L_3072_from_bin_movbe_64_end; +L_3072_from_bin_movbe_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 56))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 48))); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 40))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 32))); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 24))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 16))); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 8))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_3072_from_bin_movbe_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_3072_from_bin_movbe_64_start; + } + goto L_3072_from_bin_movbe_8_end; +L_3072_from_bin_movbe_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_3072_from_bin_movbe_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_3072_from_bin_movbe_8_start; + } + if ((rcx) == (0)) { + goto L_3072_from_bin_movbe_hi_end; + } + r8 = (word64)(0); + rax = (word64)(0); +L_3072_from_bin_movbe_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_3072_from_bin_movbe_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_3072_from_bin_movbe_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_3072_from_bin_movbe_zero_end; + } +L_3072_from_bin_movbe_zero_start: + WC_S64(rdi, 0) = (word64)(0); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_3072_from_bin_movbe_zero_start; + } +L_3072_from_bin_movbe_zero_end: + ; + (void)rsi; +} + +#endif /* !NO_MOVBE_SUPPORT */ +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 384 + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WOLFSSL_LOCAL void sp_3072_to_bin_bswap_48(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rdi, 376)); + rax = (word64)(WC_L64(rdi, 368)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 360)); + rax = (word64)(WC_L64(rdi, 352)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 344)); + rax = (word64)(WC_L64(rdi, 336)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 32) = (word64)(rdx); + WC_S64(rsi, 40) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 328)); + rax = (word64)(WC_L64(rdi, 320)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 48) = (word64)(rdx); + WC_S64(rsi, 56) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 312)); + rax = (word64)(WC_L64(rdi, 304)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 64) = (word64)(rdx); + WC_S64(rsi, 72) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 296)); + rax = (word64)(WC_L64(rdi, 288)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 80) = (word64)(rdx); + WC_S64(rsi, 88) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 280)); + rax = (word64)(WC_L64(rdi, 272)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 96) = (word64)(rdx); + WC_S64(rsi, 104) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 264)); + rax = (word64)(WC_L64(rdi, 256)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 112) = (word64)(rdx); + WC_S64(rsi, 120) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 248)); + rax = (word64)(WC_L64(rdi, 240)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 128) = (word64)(rdx); + WC_S64(rsi, 136) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 232)); + rax = (word64)(WC_L64(rdi, 224)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 144) = (word64)(rdx); + WC_S64(rsi, 152) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 216)); + rax = (word64)(WC_L64(rdi, 208)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 160) = (word64)(rdx); + WC_S64(rsi, 168) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 200)); + rax = (word64)(WC_L64(rdi, 192)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 176) = (word64)(rdx); + WC_S64(rsi, 184) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 184)); + rax = (word64)(WC_L64(rdi, 176)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 192) = (word64)(rdx); + WC_S64(rsi, 200) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 168)); + rax = (word64)(WC_L64(rdi, 160)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 208) = (word64)(rdx); + WC_S64(rsi, 216) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 152)); + rax = (word64)(WC_L64(rdi, 144)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 224) = (word64)(rdx); + WC_S64(rsi, 232) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 136)); + rax = (word64)(WC_L64(rdi, 128)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 240) = (word64)(rdx); + WC_S64(rsi, 248) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 120)); + rax = (word64)(WC_L64(rdi, 112)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 256) = (word64)(rdx); + WC_S64(rsi, 264) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 104)); + rax = (word64)(WC_L64(rdi, 96)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 272) = (word64)(rdx); + WC_S64(rsi, 280) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 88)); + rax = (word64)(WC_L64(rdi, 80)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 288) = (word64)(rdx); + WC_S64(rsi, 296) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 72)); + rax = (word64)(WC_L64(rdi, 64)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 304) = (word64)(rdx); + WC_S64(rsi, 312) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 56)); + rax = (word64)(WC_L64(rdi, 48)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 320) = (word64)(rdx); + WC_S64(rsi, 328) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 40)); + rax = (word64)(WC_L64(rdi, 32)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 336) = (word64)(rdx); + WC_S64(rsi, 344) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 24)); + rax = (word64)(WC_L64(rdi, 16)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 352) = (word64)(rdx); + WC_S64(rsi, 360) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 8)); + rax = (word64)(WC_L64(rdi, 0)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 368) = (word64)(rdx); + WC_S64(rsi, 376) = (word64)(rax); +} + +#ifndef NO_MOVBE_SUPPORT +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 384 + * Uses the movbe instruction which is optional. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_3072_to_bin_movbe_48(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 376))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 368))); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 360))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 352))); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 344))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 336))); + WC_S64(rsi, 32) = (word64)(rdx); + WC_S64(rsi, 40) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 328))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 320))); + WC_S64(rsi, 48) = (word64)(rdx); + WC_S64(rsi, 56) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 312))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 304))); + WC_S64(rsi, 64) = (word64)(rdx); + WC_S64(rsi, 72) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 296))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 288))); + WC_S64(rsi, 80) = (word64)(rdx); + WC_S64(rsi, 88) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 280))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 272))); + WC_S64(rsi, 96) = (word64)(rdx); + WC_S64(rsi, 104) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 264))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 256))); + WC_S64(rsi, 112) = (word64)(rdx); + WC_S64(rsi, 120) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 248))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 240))); + WC_S64(rsi, 128) = (word64)(rdx); + WC_S64(rsi, 136) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 232))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 224))); + WC_S64(rsi, 144) = (word64)(rdx); + WC_S64(rsi, 152) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 216))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 208))); + WC_S64(rsi, 160) = (word64)(rdx); + WC_S64(rsi, 168) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 200))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 192))); + WC_S64(rsi, 176) = (word64)(rdx); + WC_S64(rsi, 184) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 184))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 176))); + WC_S64(rsi, 192) = (word64)(rdx); + WC_S64(rsi, 200) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 168))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 160))); + WC_S64(rsi, 208) = (word64)(rdx); + WC_S64(rsi, 216) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 152))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 144))); + WC_S64(rsi, 224) = (word64)(rdx); + WC_S64(rsi, 232) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 136))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 128))); + WC_S64(rsi, 240) = (word64)(rdx); + WC_S64(rsi, 248) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 120))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 112))); + WC_S64(rsi, 256) = (word64)(rdx); + WC_S64(rsi, 264) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 104))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 96))); + WC_S64(rsi, 272) = (word64)(rdx); + WC_S64(rsi, 280) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 88))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 80))); + WC_S64(rsi, 288) = (word64)(rdx); + WC_S64(rsi, 296) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 72))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 64))); + WC_S64(rsi, 304) = (word64)(rdx); + WC_S64(rsi, 312) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 56))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 48))); + WC_S64(rsi, 320) = (word64)(rdx); + WC_S64(rsi, 328) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 40))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 32))); + WC_S64(rsi, 336) = (word64)(rdx); + WC_S64(rsi, 344) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 24))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 16))); + WC_S64(rsi, 352) = (word64)(rdx); + WC_S64(rsi, 360) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 8))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 0))); + WC_S64(rsi, 368) = (word64)(rdx); + WC_S64(rsi, 376) = (word64)(rax); +} + +#endif /* NO_MOVBE_SUPPORT */ +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_3072_mul_12(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rcx, rsp, rax, rdx = 0, r10, r9, r8; + XALIGNED(32) WC_X64I_SLOT stk[12]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 96; + rsp = (word64)(rsp - 96); + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 8) = (word64)(r9); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 16) = (word64)(r10); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 24) = (word64)(r8); + /* A[0] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 32) = (word64)(r9); + /* A[0] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 40) = (word64)(r10); + /* A[0] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 48) = (word64)(r8); + /* A[0] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 56) = (word64)(r9); + /* A[0] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 64) = (word64)(r10); + /* A[0] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 72) = (word64)(r8); + /* A[0] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 80) = (word64)(r9); + /* A[0] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 88) = (word64)(r10); + /* A[1] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 96) = (word64)(r8); + /* A[2] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + /* A[3] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r10); + /* A[4] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 120) = (word64)(r8); + /* A[5] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 128) = (word64)(r9); + /* A[6] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 136) = (word64)(r10); + /* A[7] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 144) = (word64)(r8); + /* A[8] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 152) = (word64)(r9); + /* A[9] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 160) = (word64)(r10); + /* A[10] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 168) = (word64)(r8); + /* A[11] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + WC_S64(rdi, 176) = (word64)(r9); + WC_S64(rdi, 184) = (word64)(r10); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r8 = (word64)(WC_L64(rsp, 16)); + r9 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + r8 = (word64)(WC_L64(rsp, 48)); + r9 = (word64)(WC_L64(rsp, 56)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 64)); + rdx = (word64)(WC_L64(rsp, 72)); + r8 = (word64)(WC_L64(rsp, 80)); + r9 = (word64)(WC_L64(rsp, 88)); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rdx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply. + * @param [in] b Second number to multiply. + */ +WOLFSSL_LOCAL void sp_3072_mul_avx2_12(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + sp_3072_mul_12(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_3072_add_12(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax, r8; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Add */ + rcx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(0); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Sub b from a into a. (a -= b) + * + * @param [in, out] a A single precision integer and result. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_3072_sub_in_place_24(sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rdi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 16), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 24), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 32), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 48), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 56), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 64), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 80), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 88), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 96), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 112), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 120), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 128), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 136), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 144), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 152), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 160), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 168), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 176), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 184), (unsigned long long*)&rcx); + WC_S64(rdi, 184) = (word64)(rcx); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_3072_add_24(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax, r8; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Add */ + rcx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(0); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 96)); + WC_S64(rdi, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 96), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 104)); + WC_S64(rdi, 96) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 112)); + WC_S64(rdi, 104) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 112), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 120)); + WC_S64(rdi, 112) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 120), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(rdi, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 136), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 144)); + WC_S64(rdi, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 144), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 152)); + WC_S64(rdi, 144) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 152), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 160)); + WC_S64(rdi, 152) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 160), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 168)); + WC_S64(rdi, 160) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 168), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(rdi, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 184), (unsigned long long*)&r8); + WC_S64(rdi, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_3072_mul_24(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rsp, r10, r12, rax, r13, rcx, r8, r11, r14, r15, r9; + XALIGNED(32) WC_X64I_SLOT stk[80]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 640; + rsp = (word64)(rsp - 616); + WC_S64(rsp, 576) = (word64)(rdi); + WC_S64(rsp, 584) = (word64)(rsi); + WC_S64(rsp, 592) = (word64)(rdx); + r10 = (word64)(rsp + 384); + r12 = (word64)(rsi + 96); + /* Add */ + rax = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + WC_S64(r10, 88) = (word64)(r8); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 600) = (word64)(r13); + r11 = (word64)(rsp + 480); + r12 = (word64)(rdx + 96); + /* Add */ + rax = (word64)(WC_L64(rdx, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 8)); + WC_S64(r11, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 16)); + WC_S64(r11, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 24)); + WC_S64(r11, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 32)); + WC_S64(r11, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 40)); + WC_S64(r11, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 48)); + WC_S64(r11, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 56)); + WC_S64(r11, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 64)); + WC_S64(r11, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 72)); + WC_S64(r11, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 80)); + WC_S64(r11, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 88)); + WC_S64(r11, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + WC_S64(r11, 88) = (word64)(r8); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 608) = (word64)(r14); + rdx = (word64)(r11); + rsi = (word64)(r10); + rdi = (word64)(rsp); + (void)sp_3072_mul_12((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 592)); + rsi = (word64)(WC_L64(rsp, 584)); + rdi = (word64)(rsp + 192); + rdx = (word64)(rdx + 0x60); + rsi = (word64)(rsi + 0x60); + (void)sp_3072_mul_12((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 592)); + rsi = (word64)(WC_L64(rsp, 584)); + rdi = (word64)(WC_L64(rsp, 576)); + (void)sp_3072_mul_12((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); +#ifdef _WIN64 + rdx = (word64)(WC_L64(rsp, 592)); + rsi = (word64)(WC_L64(rsp, 584)); + rdi = (word64)(WC_L64(rsp, 576)); +#endif /* _WIN64 */ + r13 = (word64)(WC_L64(rsp, 600)); + r14 = (word64)(WC_L64(rsp, 608)); + r15 = (word64)(WC_L64(rsp, 576)); + r9 = (word64)(r13); + r10 = (word64)(rsp + 384); + r11 = (word64)(rsp + 480); + r9 = (word64)(r9 & r14); + r13 = (word64)(0 - r13); + r14 = (word64)(0 - r14); + r15 = (word64)(r15 + 0xc0); + rax = (word64)(WC_L64(r10, 0)); + rcx = (word64)(WC_L64(r11, 0)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 0) = (word64)(rax); + WC_S64(r11, 0) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 8)); + rcx = (word64)(WC_L64(r11, 8)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 8) = (word64)(rax); + WC_S64(r11, 8) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 16)); + rcx = (word64)(WC_L64(r11, 16)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 16) = (word64)(rax); + WC_S64(r11, 16) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 24)); + rcx = (word64)(WC_L64(r11, 24)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 24) = (word64)(rax); + WC_S64(r11, 24) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 32)); + rcx = (word64)(WC_L64(r11, 32)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 32) = (word64)(rax); + WC_S64(r11, 32) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 40)); + rcx = (word64)(WC_L64(r11, 40)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 40) = (word64)(rax); + WC_S64(r11, 40) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 48)); + rcx = (word64)(WC_L64(r11, 48)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 48) = (word64)(rax); + WC_S64(r11, 48) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 56)); + rcx = (word64)(WC_L64(r11, 56)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 56) = (word64)(rax); + WC_S64(r11, 56) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 64)); + rcx = (word64)(WC_L64(r11, 64)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 64) = (word64)(rax); + WC_S64(r11, 64) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 72)); + rcx = (word64)(WC_L64(r11, 72)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 72) = (word64)(rax); + WC_S64(r11, 72) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 80)); + rcx = (word64)(WC_L64(r11, 80)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 80) = (word64)(rax); + WC_S64(r11, 80) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 88)); + rcx = (word64)(WC_L64(r11, 88)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 88) = (word64)(rax); + WC_S64(r11, 88) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r11 = (word64)(rsp + 192); + r10 = (word64)(rsp); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(rdi, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 184), (unsigned long long*)&r8); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + r15 = (word64)(r15 - 0x60); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r10, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 184), (unsigned long long*)&r8); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 288) = (word64)(r9); + r15 = (word64)(r15 + 0x60); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + WC_S64(r15, 96) = (word64)(rax); + /* Add to zero */ + rax = (word64)(WC_L64(r11, 104)); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 112)); + WC_S64(r15, 104) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 120)); + WC_S64(r15, 112) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 128)); + WC_S64(r15, 120) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 136)); + WC_S64(r15, 128) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 144)); + WC_S64(r15, 136) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 152)); + WC_S64(r15, 144) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 160)); + WC_S64(r15, 152) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 168)); + WC_S64(r15, 160) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 176)); + WC_S64(r15, 168) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 184)); + WC_S64(r15, 176) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(r15, 184) = (word64)(rcx); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_3072_mul_avx2_24(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rsp, r10, r12, rax, r13, rcx, r8, r11, r14, r15, r9; + XALIGNED(32) WC_X64I_SLOT stk[80]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 640; + rsp = (word64)(rsp - 616); + WC_S64(rsp, 576) = (word64)(rdi); + WC_S64(rsp, 584) = (word64)(rsi); + WC_S64(rsp, 592) = (word64)(rdx); + r10 = (word64)(rsp + 384); + r12 = (word64)(rsi + 96); + /* Add */ + rax = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + WC_S64(r10, 88) = (word64)(r8); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 600) = (word64)(r13); + r11 = (word64)(rsp + 480); + r12 = (word64)(rdx + 96); + /* Add */ + rax = (word64)(WC_L64(rdx, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 8)); + WC_S64(r11, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 16)); + WC_S64(r11, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 24)); + WC_S64(r11, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 32)); + WC_S64(r11, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 40)); + WC_S64(r11, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 48)); + WC_S64(r11, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 56)); + WC_S64(r11, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 64)); + WC_S64(r11, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 72)); + WC_S64(r11, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 80)); + WC_S64(r11, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 88)); + WC_S64(r11, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + WC_S64(r11, 88) = (word64)(r8); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 608) = (word64)(r14); + rdx = (word64)(r11); + rsi = (word64)(r10); + rdi = (word64)(rsp); + (void)sp_3072_mul_avx2_12((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 592)); + rsi = (word64)(WC_L64(rsp, 584)); + rdi = (word64)(rsp + 192); + rdx = (word64)(rdx + 0x60); + rsi = (word64)(rsi + 0x60); + (void)sp_3072_mul_avx2_12((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 592)); + rsi = (word64)(WC_L64(rsp, 584)); + rdi = (word64)(WC_L64(rsp, 576)); + (void)sp_3072_mul_avx2_12((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); +#ifdef _WIN64 + rdx = (word64)(WC_L64(rsp, 592)); + rsi = (word64)(WC_L64(rsp, 584)); + rdi = (word64)(WC_L64(rsp, 576)); +#endif /* _WIN64 */ + r13 = (word64)(WC_L64(rsp, 600)); + r14 = (word64)(WC_L64(rsp, 608)); + r15 = (word64)(WC_L64(rsp, 576)); + r9 = (word64)(r13); + r10 = (word64)(rsp + 384); + r11 = (word64)(rsp + 480); + r9 = (word64)(r9 & r14); + r13 = (word64)(0 - r13); + r14 = (word64)(0 - r14); + r15 = (word64)(r15 + 0xc0); + rax = (word64)(WC_L64(r10, 0)); + rcx = (word64)(WC_L64(r11, 0)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + r8 = (word64)(WC_L64(r11, 8)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + rax = (word64)(WC_L64(r11, 16)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + rcx = (word64)(WC_L64(r11, 24)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + r8 = (word64)(WC_L64(r11, 32)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + rax = (word64)(WC_L64(r11, 40)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + rcx = (word64)(WC_L64(r11, 48)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + r8 = (word64)(WC_L64(r11, 56)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + rax = (word64)(WC_L64(r11, 64)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + rcx = (word64)(WC_L64(r11, 72)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + r8 = (word64)(WC_L64(r11, 80)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + rax = (word64)(WC_L64(r11, 88)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r11 = (word64)(rsp + 192); + r10 = (word64)(rsp); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(rdi, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 184), (unsigned long long*)&r8); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + r15 = (word64)(r15 - 0x60); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r10, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 184), (unsigned long long*)&r8); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 288) = (word64)(r9); + r15 = (word64)(r15 + 0x60); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + WC_S64(r15, 96) = (word64)(rax); + /* Add to zero */ + rax = (word64)(WC_L64(r11, 104)); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 112)); + WC_S64(r15, 104) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 120)); + WC_S64(r15, 112) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 128)); + WC_S64(r15, 120) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 136)); + WC_S64(r15, 128) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 144)); + WC_S64(r15, 136) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 152)); + WC_S64(r15, 144) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 160)); + WC_S64(r15, 152) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 168)); + WC_S64(r15, 160) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 176)); + WC_S64(r15, 168) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 184)); + WC_S64(r15, 176) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(r15, 184) = (word64)(rcx); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Sub b from a into a. (a -= b) + * + * @param [in, out] a A single precision integer and result. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_3072_sub_in_place_48(sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rdi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 16), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 24), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 32), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 48), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 56), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 64), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 80), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 88), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 96), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 112), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 120), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 128), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 136), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 144), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 152), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 160), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 168), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 176), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 184), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 192)); + WC_S64(rdi, 184) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 192), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 200)); + WC_S64(rdi, 192) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 200), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 208)); + WC_S64(rdi, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 208), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 216)); + WC_S64(rdi, 208) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 216), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 224)); + WC_S64(rdi, 216) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 224), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 232)); + WC_S64(rdi, 224) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 232), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 240)); + WC_S64(rdi, 232) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 240), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 248)); + WC_S64(rdi, 240) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 248), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 256)); + WC_S64(rdi, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 256), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 264)); + WC_S64(rdi, 256) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 264), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 272)); + WC_S64(rdi, 264) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 272), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 280)); + WC_S64(rdi, 272) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 280), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 288)); + WC_S64(rdi, 280) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 288), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 296)); + WC_S64(rdi, 288) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 296), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 304)); + WC_S64(rdi, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 304), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 312)); + WC_S64(rdi, 304) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 312), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 320)); + WC_S64(rdi, 312) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 320), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 328)); + WC_S64(rdi, 320) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 328), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 336)); + WC_S64(rdi, 328) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 336), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 344)); + WC_S64(rdi, 336) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 344), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 352)); + WC_S64(rdi, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 352), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 360)); + WC_S64(rdi, 352) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 360), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 368)); + WC_S64(rdi, 360) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 368), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 376)); + WC_S64(rdi, 368) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 376), (unsigned long long*)&rcx); + WC_S64(rdi, 376) = (word64)(rcx); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_3072_add_48(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax, r8; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Add */ + rcx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(0); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 96)); + WC_S64(rdi, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 96), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 104)); + WC_S64(rdi, 96) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 112)); + WC_S64(rdi, 104) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 112), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 120)); + WC_S64(rdi, 112) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 120), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(rdi, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 136), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 144)); + WC_S64(rdi, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 144), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 152)); + WC_S64(rdi, 144) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 152), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 160)); + WC_S64(rdi, 152) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 160), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 168)); + WC_S64(rdi, 160) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 168), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(rdi, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 184), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 192)); + WC_S64(rdi, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 192), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 200)); + WC_S64(rdi, 192) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 200), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 208)); + WC_S64(rdi, 200) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 208), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 216)); + WC_S64(rdi, 208) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 216), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 224)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 232)); + WC_S64(rdi, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 232), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 240)); + WC_S64(rdi, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 240), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 248)); + WC_S64(rdi, 240) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 248), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 256)); + WC_S64(rdi, 248) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 256), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 264)); + WC_S64(rdi, 256) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 264), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 272)); + WC_S64(rdi, 264) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 280)); + WC_S64(rdi, 272) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 280), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 288)); + WC_S64(rdi, 280) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 288), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 296)); + WC_S64(rdi, 288) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 296), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 304)); + WC_S64(rdi, 296) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 304), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 312)); + WC_S64(rdi, 304) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 312), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 320)); + WC_S64(rdi, 312) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 328)); + WC_S64(rdi, 320) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 328), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 336)); + WC_S64(rdi, 328) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 336), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 344)); + WC_S64(rdi, 336) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 344), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 352)); + WC_S64(rdi, 344) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 352), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 360)); + WC_S64(rdi, 352) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 360), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 368)); + WC_S64(rdi, 360) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 376)); + WC_S64(rdi, 368) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 376), (unsigned long long*)&r8); + WC_S64(rdi, 376) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_3072_mul_48(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rsp, r10, r12, rax, r13, rcx, r8, r11, r14, r15, r9; + XALIGNED(32) WC_X64I_SLOT stk[152]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 1216; + rsp = (word64)(rsp - 1192); + WC_S64(rsp, 1152) = (word64)(rdi); + WC_S64(rsp, 1160) = (word64)(rsi); + WC_S64(rsp, 1168) = (word64)(rdx); + r10 = (word64)(rsp + 768); + r12 = (word64)(rsi + 192); + /* Add */ + rax = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 184), (unsigned long long*)&r8); + WC_S64(r10, 184) = (word64)(r8); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 1176) = (word64)(r13); + r11 = (word64)(rsp + 960); + r12 = (word64)(rdx + 192); + /* Add */ + rax = (word64)(WC_L64(rdx, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 8)); + WC_S64(r11, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 16)); + WC_S64(r11, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 24)); + WC_S64(r11, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 32)); + WC_S64(r11, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 40)); + WC_S64(r11, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 48)); + WC_S64(r11, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 56)); + WC_S64(r11, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 64)); + WC_S64(r11, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 72)); + WC_S64(r11, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 80)); + WC_S64(r11, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 88)); + WC_S64(r11, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 96)); + WC_S64(r11, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 104)); + WC_S64(r11, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 112)); + WC_S64(r11, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 120)); + WC_S64(r11, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 128)); + WC_S64(r11, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 136)); + WC_S64(r11, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 144)); + WC_S64(r11, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 152)); + WC_S64(r11, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 160)); + WC_S64(r11, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 168)); + WC_S64(r11, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 176)); + WC_S64(r11, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 184)); + WC_S64(r11, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 184), (unsigned long long*)&r8); + WC_S64(r11, 184) = (word64)(r8); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 1184) = (word64)(r14); + rdx = (word64)(r11); + rsi = (word64)(r10); + rdi = (word64)(rsp); + (void)sp_3072_mul_24((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 1168)); + rsi = (word64)(WC_L64(rsp, 1160)); + rdi = (word64)(rsp + 384); + rdx = (word64)(rdx + 0xc0); + rsi = (word64)(rsi + 0xc0); + (void)sp_3072_mul_24((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 1168)); + rsi = (word64)(WC_L64(rsp, 1160)); + rdi = (word64)(WC_L64(rsp, 1152)); + (void)sp_3072_mul_24((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); +#ifdef _WIN64 + rdx = (word64)(WC_L64(rsp, 1168)); + rsi = (word64)(WC_L64(rsp, 1160)); + rdi = (word64)(WC_L64(rsp, 1152)); +#endif /* _WIN64 */ + r13 = (word64)(WC_L64(rsp, 1176)); + r14 = (word64)(WC_L64(rsp, 1184)); + r15 = (word64)(WC_L64(rsp, 1152)); + r9 = (word64)(r13); + r10 = (word64)(rsp + 768); + r11 = (word64)(rsp + 960); + r9 = (word64)(r9 & r14); + r13 = (word64)(0 - r13); + r14 = (word64)(0 - r14); + r15 = (word64)(r15 + 0x180); + rax = (word64)(WC_L64(r10, 0)); + rcx = (word64)(WC_L64(r11, 0)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 0) = (word64)(rax); + WC_S64(r11, 0) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 8)); + rcx = (word64)(WC_L64(r11, 8)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 8) = (word64)(rax); + WC_S64(r11, 8) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 16)); + rcx = (word64)(WC_L64(r11, 16)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 16) = (word64)(rax); + WC_S64(r11, 16) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 24)); + rcx = (word64)(WC_L64(r11, 24)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 24) = (word64)(rax); + WC_S64(r11, 24) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 32)); + rcx = (word64)(WC_L64(r11, 32)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 32) = (word64)(rax); + WC_S64(r11, 32) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 40)); + rcx = (word64)(WC_L64(r11, 40)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 40) = (word64)(rax); + WC_S64(r11, 40) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 48)); + rcx = (word64)(WC_L64(r11, 48)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 48) = (word64)(rax); + WC_S64(r11, 48) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 56)); + rcx = (word64)(WC_L64(r11, 56)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 56) = (word64)(rax); + WC_S64(r11, 56) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 64)); + rcx = (word64)(WC_L64(r11, 64)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 64) = (word64)(rax); + WC_S64(r11, 64) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 72)); + rcx = (word64)(WC_L64(r11, 72)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 72) = (word64)(rax); + WC_S64(r11, 72) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 80)); + rcx = (word64)(WC_L64(r11, 80)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 80) = (word64)(rax); + WC_S64(r11, 80) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 88)); + rcx = (word64)(WC_L64(r11, 88)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 88) = (word64)(rax); + WC_S64(r11, 88) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 96)); + rcx = (word64)(WC_L64(r11, 96)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 96) = (word64)(rax); + WC_S64(r11, 96) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 104)); + rcx = (word64)(WC_L64(r11, 104)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 104) = (word64)(rax); + WC_S64(r11, 104) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 112)); + rcx = (word64)(WC_L64(r11, 112)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 112) = (word64)(rax); + WC_S64(r11, 112) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 120)); + rcx = (word64)(WC_L64(r11, 120)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 120) = (word64)(rax); + WC_S64(r11, 120) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 128)); + rcx = (word64)(WC_L64(r11, 128)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 128) = (word64)(rax); + WC_S64(r11, 128) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 136)); + rcx = (word64)(WC_L64(r11, 136)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 136) = (word64)(rax); + WC_S64(r11, 136) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 144)); + rcx = (word64)(WC_L64(r11, 144)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 144) = (word64)(rax); + WC_S64(r11, 144) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 152)); + rcx = (word64)(WC_L64(r11, 152)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 152) = (word64)(rax); + WC_S64(r11, 152) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 160)); + rcx = (word64)(WC_L64(r11, 160)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 160) = (word64)(rax); + WC_S64(r11, 160) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 168)); + rcx = (word64)(WC_L64(r11, 168)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 168) = (word64)(rax); + WC_S64(r11, 168) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 176)); + rcx = (word64)(WC_L64(r11, 176)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 176) = (word64)(rax); + WC_S64(r11, 176) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 184)); + rcx = (word64)(WC_L64(r11, 184)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 184) = (word64)(rax); + WC_S64(r11, 184) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r11 = (word64)(rsp + 384); + r10 = (word64)(rsp); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 256)); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 264)); + WC_S64(r10, 256) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 272)); + WC_S64(r10, 264) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 280)); + WC_S64(r10, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 288)); + WC_S64(r10, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 296)); + WC_S64(r10, 288) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 304)); + WC_S64(r10, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 312)); + WC_S64(r10, 304) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 320)); + WC_S64(r10, 312) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 328)); + WC_S64(r10, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 336)); + WC_S64(r10, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 344)); + WC_S64(r10, 336) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 352)); + WC_S64(r10, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 360)); + WC_S64(r10, 352) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 368)); + WC_S64(r10, 360) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 376)); + WC_S64(r10, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 376), (unsigned long long*)&r8); + WC_S64(r10, 376) = (word64)(r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(rdi, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 256)); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 264)); + WC_S64(r10, 256) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 272)); + WC_S64(r10, 264) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 280)); + WC_S64(r10, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 288)); + WC_S64(r10, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 296)); + WC_S64(r10, 288) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 304)); + WC_S64(r10, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 312)); + WC_S64(r10, 304) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 320)); + WC_S64(r10, 312) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 328)); + WC_S64(r10, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 336)); + WC_S64(r10, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 344)); + WC_S64(r10, 336) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 352)); + WC_S64(r10, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 360)); + WC_S64(r10, 352) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 368)); + WC_S64(r10, 360) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 376)); + WC_S64(r10, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 376), (unsigned long long*)&r8); + WC_S64(r10, 376) = (word64)(r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + r15 = (word64)(r15 - 0xc0); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r10, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 200)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 208)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 216)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 224)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 232)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 240)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 248)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 256)); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 264)); + WC_S64(r15, 256) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 272)); + WC_S64(r15, 264) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 280)); + WC_S64(r15, 272) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 288)); + WC_S64(r15, 280) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 296)); + WC_S64(r15, 288) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 304)); + WC_S64(r15, 296) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 312)); + WC_S64(r15, 304) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 320)); + WC_S64(r15, 312) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 328)); + WC_S64(r15, 320) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 336)); + WC_S64(r15, 328) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 344)); + WC_S64(r15, 336) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 352)); + WC_S64(r15, 344) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 360)); + WC_S64(r15, 352) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 368)); + WC_S64(r15, 360) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 376)); + WC_S64(r15, 368) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 376), (unsigned long long*)&r8); + WC_S64(r15, 376) = (word64)(r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 576) = (word64)(r9); + r15 = (word64)(r15 + 0xc0); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + WC_S64(r15, 192) = (word64)(rax); + /* Add to zero */ + rax = (word64)(WC_L64(r11, 200)); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 208)); + WC_S64(r15, 200) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 216)); + WC_S64(r15, 208) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 224)); + WC_S64(r15, 216) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 232)); + WC_S64(r15, 224) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 240)); + WC_S64(r15, 232) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 248)); + WC_S64(r15, 240) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 256)); + WC_S64(r15, 248) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 264)); + WC_S64(r15, 256) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 272)); + WC_S64(r15, 264) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 280)); + WC_S64(r15, 272) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 288)); + WC_S64(r15, 280) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 296)); + WC_S64(r15, 288) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 304)); + WC_S64(r15, 296) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 312)); + WC_S64(r15, 304) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 320)); + WC_S64(r15, 312) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 328)); + WC_S64(r15, 320) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 336)); + WC_S64(r15, 328) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 344)); + WC_S64(r15, 336) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 352)); + WC_S64(r15, 344) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 360)); + WC_S64(r15, 352) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 368)); + WC_S64(r15, 360) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 376)); + WC_S64(r15, 368) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(r15, 376) = (word64)(rcx); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_3072_mul_avx2_48(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rsp, r10, r12, rax, r13, rcx, r8, r11, r14, r15, r9; + XALIGNED(32) WC_X64I_SLOT stk[152]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 1216; + rsp = (word64)(rsp - 1192); + WC_S64(rsp, 1152) = (word64)(rdi); + WC_S64(rsp, 1160) = (word64)(rsi); + WC_S64(rsp, 1168) = (word64)(rdx); + r10 = (word64)(rsp + 768); + r12 = (word64)(rsi + 192); + /* Add */ + rax = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 184), (unsigned long long*)&r8); + WC_S64(r10, 184) = (word64)(r8); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 1176) = (word64)(r13); + r11 = (word64)(rsp + 960); + r12 = (word64)(rdx + 192); + /* Add */ + rax = (word64)(WC_L64(rdx, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 8)); + WC_S64(r11, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 16)); + WC_S64(r11, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 24)); + WC_S64(r11, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 32)); + WC_S64(r11, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 40)); + WC_S64(r11, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 48)); + WC_S64(r11, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 56)); + WC_S64(r11, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 64)); + WC_S64(r11, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 72)); + WC_S64(r11, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 80)); + WC_S64(r11, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 88)); + WC_S64(r11, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 96)); + WC_S64(r11, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 104)); + WC_S64(r11, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 112)); + WC_S64(r11, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 120)); + WC_S64(r11, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 128)); + WC_S64(r11, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 136)); + WC_S64(r11, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 144)); + WC_S64(r11, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 152)); + WC_S64(r11, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 160)); + WC_S64(r11, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 168)); + WC_S64(r11, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 176)); + WC_S64(r11, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 184)); + WC_S64(r11, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 184), (unsigned long long*)&r8); + WC_S64(r11, 184) = (word64)(r8); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 1184) = (word64)(r14); + rdx = (word64)(r11); + rsi = (word64)(r10); + rdi = (word64)(rsp); + (void)sp_3072_mul_avx2_24((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 1168)); + rsi = (word64)(WC_L64(rsp, 1160)); + rdi = (word64)(rsp + 384); + rdx = (word64)(rdx + 0xc0); + rsi = (word64)(rsi + 0xc0); + (void)sp_3072_mul_avx2_24((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 1168)); + rsi = (word64)(WC_L64(rsp, 1160)); + rdi = (word64)(WC_L64(rsp, 1152)); + (void)sp_3072_mul_avx2_24((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); +#ifdef _WIN64 + rdx = (word64)(WC_L64(rsp, 1168)); + rsi = (word64)(WC_L64(rsp, 1160)); + rdi = (word64)(WC_L64(rsp, 1152)); +#endif /* _WIN64 */ + r13 = (word64)(WC_L64(rsp, 1176)); + r14 = (word64)(WC_L64(rsp, 1184)); + r15 = (word64)(WC_L64(rsp, 1152)); + r9 = (word64)(r13); + r10 = (word64)(rsp + 768); + r11 = (word64)(rsp + 960); + r9 = (word64)(r9 & r14); + r13 = (word64)(0 - r13); + r14 = (word64)(0 - r14); + r15 = (word64)(r15 + 0x180); + rax = (word64)(WC_L64(r10, 0)); + rcx = (word64)(WC_L64(r11, 0)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + r8 = (word64)(WC_L64(r11, 8)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + rax = (word64)(WC_L64(r11, 16)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + rcx = (word64)(WC_L64(r11, 24)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + r8 = (word64)(WC_L64(r11, 32)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + rax = (word64)(WC_L64(r11, 40)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + rcx = (word64)(WC_L64(r11, 48)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + r8 = (word64)(WC_L64(r11, 56)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + rax = (word64)(WC_L64(r11, 64)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + rcx = (word64)(WC_L64(r11, 72)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + r8 = (word64)(WC_L64(r11, 80)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + rax = (word64)(WC_L64(r11, 88)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + rcx = (word64)(WC_L64(r11, 96)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + r8 = (word64)(WC_L64(r11, 104)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + rax = (word64)(WC_L64(r11, 112)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + rcx = (word64)(WC_L64(r11, 120)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + r8 = (word64)(WC_L64(r11, 128)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + rax = (word64)(WC_L64(r11, 136)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + rcx = (word64)(WC_L64(r11, 144)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + r8 = (word64)(WC_L64(r11, 152)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + rax = (word64)(WC_L64(r11, 160)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + rcx = (word64)(WC_L64(r11, 168)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + r8 = (word64)(WC_L64(r11, 176)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + rax = (word64)(WC_L64(r11, 184)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r11 = (word64)(rsp + 384); + r10 = (word64)(rsp); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 256)); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 264)); + WC_S64(r10, 256) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 272)); + WC_S64(r10, 264) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 280)); + WC_S64(r10, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 288)); + WC_S64(r10, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 296)); + WC_S64(r10, 288) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 304)); + WC_S64(r10, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 312)); + WC_S64(r10, 304) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 320)); + WC_S64(r10, 312) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 328)); + WC_S64(r10, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 336)); + WC_S64(r10, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 344)); + WC_S64(r10, 336) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 352)); + WC_S64(r10, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 360)); + WC_S64(r10, 352) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 368)); + WC_S64(r10, 360) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 376)); + WC_S64(r10, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 376), (unsigned long long*)&r8); + WC_S64(r10, 376) = (word64)(r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(rdi, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 256)); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 264)); + WC_S64(r10, 256) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 272)); + WC_S64(r10, 264) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 280)); + WC_S64(r10, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 288)); + WC_S64(r10, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 296)); + WC_S64(r10, 288) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 304)); + WC_S64(r10, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 312)); + WC_S64(r10, 304) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 320)); + WC_S64(r10, 312) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 328)); + WC_S64(r10, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 336)); + WC_S64(r10, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 344)); + WC_S64(r10, 336) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 352)); + WC_S64(r10, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 360)); + WC_S64(r10, 352) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 368)); + WC_S64(r10, 360) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 376)); + WC_S64(r10, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 376), (unsigned long long*)&r8); + WC_S64(r10, 376) = (word64)(r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + r15 = (word64)(r15 - 0xc0); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r10, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 200)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 208)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 216)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 224)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 232)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 240)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 248)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 256)); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 264)); + WC_S64(r15, 256) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 272)); + WC_S64(r15, 264) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 280)); + WC_S64(r15, 272) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 288)); + WC_S64(r15, 280) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 296)); + WC_S64(r15, 288) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 304)); + WC_S64(r15, 296) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 312)); + WC_S64(r15, 304) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 320)); + WC_S64(r15, 312) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 328)); + WC_S64(r15, 320) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 336)); + WC_S64(r15, 328) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 344)); + WC_S64(r15, 336) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 352)); + WC_S64(r15, 344) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 360)); + WC_S64(r15, 352) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 368)); + WC_S64(r15, 360) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 376)); + WC_S64(r15, 368) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 376), (unsigned long long*)&r8); + WC_S64(r15, 376) = (word64)(r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 576) = (word64)(r9); + r15 = (word64)(r15 + 0xc0); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + WC_S64(r15, 192) = (word64)(rax); + /* Add to zero */ + rax = (word64)(WC_L64(r11, 200)); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 208)); + WC_S64(r15, 200) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 216)); + WC_S64(r15, 208) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 224)); + WC_S64(r15, 216) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 232)); + WC_S64(r15, 224) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 240)); + WC_S64(r15, 232) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 248)); + WC_S64(r15, 240) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 256)); + WC_S64(r15, 248) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 264)); + WC_S64(r15, 256) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 272)); + WC_S64(r15, 264) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 280)); + WC_S64(r15, 272) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 288)); + WC_S64(r15, 280) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 296)); + WC_S64(r15, 288) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 304)); + WC_S64(r15, 296) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 312)); + WC_S64(r15, 304) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 320)); + WC_S64(r15, 312) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 328)); + WC_S64(r15, 320) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 336)); + WC_S64(r15, 328) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 344)); + WC_S64(r15, 336) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 352)); + WC_S64(r15, 344) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 360)); + WC_S64(r15, 352) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 368)); + WC_S64(r15, 360) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 376)); + WC_S64(r15, 368) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(r15, 376) = (word64)(rcx); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_3072_sqr_12(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rax, rdx = 0, r9, r8, rcx, r12, r10, r11; + XALIGNED(32) WC_X64I_SLOT stk[12]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 96; + rsp = (word64)(rsp - 96); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 8) = (word64)(r8); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 16) = (word64)(r9); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 24) = (word64)(rcx); + /* A[0] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 32) = (word64)(r8); + /* A[0] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 40) = (word64)(r9); + /* A[0] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 48) = (word64)(rcx); + /* A[0] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rsp, 56) = (word64)(r8); + /* A[0] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 64) = (word64)(r9); + /* A[0] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 72) = (word64)(rcx); + /* A[0] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rsp, 80) = (word64)(r8); + /* A[0] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 88) = (word64)(r9); + /* A[1] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[2] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(rcx); + /* A[2] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[3] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 104) = (word64)(r8); + /* A[3] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[4] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rdi, 112) = (word64)(r9); + /* A[4] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[5] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rdi, 120) = (word64)(rcx); + /* A[5] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[6] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 128) = (word64)(r8); + /* A[6] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[7] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rdi, 136) = (word64)(r9); + /* A[7] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 144) = (word64)(rcx); + /* A[8] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[9] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 152) = (word64)(r8); + /* A[9] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 160) = (word64)(r9); + /* A[10] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 168) = (word64)(rcx); + /* A[11] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r8); + WC_S64(rdi, 184) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r10 = (word64)(WC_L64(rsp, 16)); + r11 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + r10 = (word64)(WC_L64(rsp, 48)); + r11 = (word64)(WC_L64(rsp, 56)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(r10); + WC_S64(rdi, 56) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 64)); + rdx = (word64)(WC_L64(rsp, 72)); + r10 = (word64)(WC_L64(rsp, 80)); + r11 = (word64)(WC_L64(rsp, 88)); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rdx); + WC_S64(rdi, 80) = (word64)(r10); + WC_S64(rdi, 88) = (word64)(r11); +} + +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_3072_sqr_avx2_12(sp_digit* r, const sp_digit* a) +{ + sp_3072_sqr_12(r, a); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * Karatsuba: ah^2, al^2, (al - ah)^2 + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_3072_sqr_24(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rcx, r8, r9, rdx, rax; + XALIGNED(32) WC_X64I_SLOT stk[28]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 224; + rsp = (word64)(rsp - 208); + WC_S64(rsp, 192) = (word64)(rdi); + WC_S64(rsp, 200) = (word64)(rsi); + rcx = (word64)(0); + r8 = (word64)(rsp); + r9 = (word64)(rsi + 96); + rdx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(r9, 0), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 8)); + WC_S64(r8, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 8), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 16)); + WC_S64(r8, 8) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 16), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r8, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 24), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 32)); + WC_S64(r8, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 32), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 40)); + WC_S64(r8, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 40), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 48)); + WC_S64(r8, 40) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 48), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 56)); + WC_S64(r8, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 56), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 64)); + WC_S64(r8, 56) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 64), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r8, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 72), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 80)); + WC_S64(r8, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 80), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 88)); + WC_S64(r8, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 88), (unsigned long long*)&rax); + WC_S64(r8, 88) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* Cond Negate */ + rdx = (word64)(WC_L64(r8, 0)); + r9 = (word64)(rcx); + rdx = (word64)(rdx ^ rcx); + r9 = (word64)(0 - r9); + cf = _subborrow_u64(0, rdx, rcx, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 8)); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(r8, 0) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 16)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 8) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 24)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 16) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 32)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 24) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 40)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 32) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 48)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 40) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 56)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 48) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 64)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 56) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 72)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 64) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 80)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 72) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 88)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 80) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + rax = (word64)(rax + r9); + WC_S64(r8, 88) = (word64)(rax); + rsi = (word64)(r8); + rdi = (word64)(rsp); + (void)sp_3072_sqr_12((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); + rsi = (word64)(WC_L64(rsp, 200)); + rdi = (word64)(WC_L64(rsp, 192)); + rsi = (word64)(rsi + 0x60); + rdi = (word64)(rdi + 0xc0); + (void)sp_3072_sqr_12((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); + rsi = (word64)(WC_L64(rsp, 200)); + rdi = (word64)(WC_L64(rsp, 192)); + (void)sp_3072_sqr_12((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); +#ifdef _WIN64 + rsi = (word64)(WC_L64(rsp, 200)); + rdi = (word64)(WC_L64(rsp, 192)); +#endif /* _WIN64 */ + rsi = (word64)(WC_L64(rsp, 192)); + r8 = (word64)(rsp + 96); + rsi = (word64)(rsi + 0x120); + rcx = (word64)(0); + rax = (word64)(WC_L64(r8, -96)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rsi = (word64)(rsi - 0xc0); + rax = (word64)(WC_L64(r8, -96)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 192)); + rcx = (word64)(0 - rcx); + rdi = (word64)(rdi + 0xc0); + rax = (word64)(WC_L64(rdi, -96)); + cf = _subborrow_u64(0, rax, WC_L64(r8, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -88)); + WC_S64(rdi, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -80)); + WC_S64(rdi, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -72)); + WC_S64(rdi, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -64)); + WC_S64(rdi, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -56)); + WC_S64(rdi, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -48)); + WC_S64(rdi, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -40)); + WC_S64(rdi, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -32)); + WC_S64(rdi, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -24)); + WC_S64(rdi, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -16)); + WC_S64(rdi, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -8)); + WC_S64(rdi, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 0)); + WC_S64(rdi, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 88), (unsigned long long*)&rdx); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 192)); + rdi = (word64)(rdi + 0x120); + /* Add in word */ + rax = (word64)(WC_L64(rdi, 0)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 88) = (word64)(rdx); + rsi = (word64)(WC_L64(rsp, 200)); + rdi = (word64)(WC_L64(rsp, 192)); +} + +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * Karatsuba: ah^2, al^2, (al - ah)^2 + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_3072_sqr_avx2_24(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rcx, r8, r9, rdx, rax; + XALIGNED(32) WC_X64I_SLOT stk[28]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 224; + rsp = (word64)(rsp - 208); + WC_S64(rsp, 192) = (word64)(rdi); + WC_S64(rsp, 200) = (word64)(rsi); + rcx = (word64)(0); + r8 = (word64)(rsp); + r9 = (word64)(rsi + 96); + rdx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(r9, 0), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 8)); + WC_S64(r8, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 8), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 16)); + WC_S64(r8, 8) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 16), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r8, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 24), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 32)); + WC_S64(r8, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 32), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 40)); + WC_S64(r8, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 40), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 48)); + WC_S64(r8, 40) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 48), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 56)); + WC_S64(r8, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 56), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 64)); + WC_S64(r8, 56) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 64), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r8, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 72), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 80)); + WC_S64(r8, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 80), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 88)); + WC_S64(r8, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 88), (unsigned long long*)&rax); + WC_S64(r8, 88) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* Cond Negate */ + rdx = (word64)(WC_L64(r8, 0)); + r9 = (word64)(rcx); + rdx = (word64)(rdx ^ rcx); + r9 = (word64)(0 - r9); + cf = _subborrow_u64(0, rdx, rcx, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 8)); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(r8, 0) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 16)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 8) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 24)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 16) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 32)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 24) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 40)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 32) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 48)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 40) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 56)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 48) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 64)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 56) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 72)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 64) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 80)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 72) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 88)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 80) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + rax = (word64)(rax + r9); + WC_S64(r8, 88) = (word64)(rax); + rsi = (word64)(r8); + rdi = (word64)(rsp); + (void)sp_3072_sqr_avx2_12((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); + rsi = (word64)(WC_L64(rsp, 200)); + rdi = (word64)(WC_L64(rsp, 192)); + rsi = (word64)(rsi + 0x60); + rdi = (word64)(rdi + 0xc0); + (void)sp_3072_sqr_avx2_12((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); + rsi = (word64)(WC_L64(rsp, 200)); + rdi = (word64)(WC_L64(rsp, 192)); + (void)sp_3072_sqr_avx2_12((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); +#ifdef _WIN64 + rsi = (word64)(WC_L64(rsp, 200)); + rdi = (word64)(WC_L64(rsp, 192)); +#endif /* _WIN64 */ + rsi = (word64)(WC_L64(rsp, 192)); + r8 = (word64)(rsp + 96); + rsi = (word64)(rsi + 0x120); + rcx = (word64)(0); + rax = (word64)(WC_L64(r8, -96)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rsi = (word64)(rsi - 0xc0); + rax = (word64)(WC_L64(r8, -96)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 192)); + rcx = (word64)(0 - rcx); + rdi = (word64)(rdi + 0xc0); + rax = (word64)(WC_L64(rdi, -96)); + cf = _subborrow_u64(0, rax, WC_L64(r8, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -88)); + WC_S64(rdi, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -80)); + WC_S64(rdi, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -72)); + WC_S64(rdi, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -64)); + WC_S64(rdi, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -56)); + WC_S64(rdi, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -48)); + WC_S64(rdi, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -40)); + WC_S64(rdi, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -32)); + WC_S64(rdi, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -24)); + WC_S64(rdi, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -16)); + WC_S64(rdi, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -8)); + WC_S64(rdi, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 0)); + WC_S64(rdi, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 88), (unsigned long long*)&rdx); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 192)); + rdi = (word64)(rdi + 0x120); + /* Add in word */ + rax = (word64)(WC_L64(rdi, 0)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 88) = (word64)(rdx); + rsi = (word64)(WC_L64(rsp, 200)); + rdi = (word64)(WC_L64(rsp, 192)); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * Karatsuba: ah^2, al^2, (al - ah)^2 + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_3072_sqr_48(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rcx, r8, r9, rdx, rax; + XALIGNED(32) WC_X64I_SLOT stk[52]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 416; + rsp = (word64)(rsp - 400); + WC_S64(rsp, 384) = (word64)(rdi); + WC_S64(rsp, 392) = (word64)(rsi); + rcx = (word64)(0); + r8 = (word64)(rsp); + r9 = (word64)(rsi + 192); + rdx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(r9, 0), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 8)); + WC_S64(r8, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 8), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 16)); + WC_S64(r8, 8) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 16), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r8, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 24), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 32)); + WC_S64(r8, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 32), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 40)); + WC_S64(r8, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 40), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 48)); + WC_S64(r8, 40) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 48), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 56)); + WC_S64(r8, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 56), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 64)); + WC_S64(r8, 56) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 64), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r8, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 72), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 80)); + WC_S64(r8, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 80), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 88)); + WC_S64(r8, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 88), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 96)); + WC_S64(r8, 88) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 96), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 104)); + WC_S64(r8, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 104), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 112)); + WC_S64(r8, 104) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 112), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r8, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 120), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 128)); + WC_S64(r8, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 128), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 136)); + WC_S64(r8, 128) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 136), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 144)); + WC_S64(r8, 136) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 144), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 152)); + WC_S64(r8, 144) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 152), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 160)); + WC_S64(r8, 152) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 160), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 168)); + WC_S64(r8, 160) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 168), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 176)); + WC_S64(r8, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 176), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 184)); + WC_S64(r8, 176) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 184), (unsigned long long*)&rax); + WC_S64(r8, 184) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* Cond Negate */ + rdx = (word64)(WC_L64(r8, 0)); + r9 = (word64)(rcx); + rdx = (word64)(rdx ^ rcx); + r9 = (word64)(0 - r9); + cf = _subborrow_u64(0, rdx, rcx, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 8)); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(r8, 0) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 16)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 8) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 24)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 16) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 32)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 24) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 40)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 32) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 48)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 40) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 56)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 48) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 64)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 56) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 72)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 64) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 80)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 72) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 88)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 80) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 96)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 88) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 104)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 96) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 112)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 104) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 120)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 112) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 128)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 120) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 136)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 128) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 144)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 136) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 152)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 144) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 160)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 152) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 168)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 160) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 176)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 168) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 184)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 176) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + rax = (word64)(rax + r9); + WC_S64(r8, 184) = (word64)(rax); + rsi = (word64)(r8); + rdi = (word64)(rsp); + (void)sp_3072_sqr_24((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); + rsi = (word64)(WC_L64(rsp, 392)); + rdi = (word64)(WC_L64(rsp, 384)); + rsi = (word64)(rsi + 0xc0); + rdi = (word64)(rdi + 0x180); + (void)sp_3072_sqr_24((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); + rsi = (word64)(WC_L64(rsp, 392)); + rdi = (word64)(WC_L64(rsp, 384)); + (void)sp_3072_sqr_24((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); +#ifdef _WIN64 + rsi = (word64)(WC_L64(rsp, 392)); + rdi = (word64)(WC_L64(rsp, 384)); +#endif /* _WIN64 */ + rsi = (word64)(WC_L64(rsp, 384)); + r8 = (word64)(rsp + 192); + rsi = (word64)(rsi + 0x240); + rcx = (word64)(0); + rax = (word64)(WC_L64(r8, -192)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -184)); + WC_S64(r8, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -176)); + WC_S64(r8, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -168)); + WC_S64(r8, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -160)); + WC_S64(r8, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -152)); + WC_S64(r8, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -144)); + WC_S64(r8, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -136)); + WC_S64(r8, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -128)); + WC_S64(r8, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 128)); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 136)); + WC_S64(r8, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 144)); + WC_S64(r8, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 152)); + WC_S64(r8, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 160)); + WC_S64(r8, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 168)); + WC_S64(r8, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 176)); + WC_S64(r8, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 184)); + WC_S64(r8, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 184), (unsigned long long*)&rdx); + WC_S64(r8, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rsi = (word64)(rsi - 0x180); + rax = (word64)(WC_L64(r8, -192)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -184)); + WC_S64(r8, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -176)); + WC_S64(r8, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -168)); + WC_S64(r8, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -160)); + WC_S64(r8, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -152)); + WC_S64(r8, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -144)); + WC_S64(r8, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -136)); + WC_S64(r8, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -128)); + WC_S64(r8, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 128)); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 136)); + WC_S64(r8, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 144)); + WC_S64(r8, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 152)); + WC_S64(r8, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 160)); + WC_S64(r8, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 168)); + WC_S64(r8, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 176)); + WC_S64(r8, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 184)); + WC_S64(r8, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 184), (unsigned long long*)&rdx); + WC_S64(r8, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 384)); + rcx = (word64)(0 - rcx); + rdi = (word64)(rdi + 0x180); + rax = (word64)(WC_L64(rdi, -192)); + cf = _subborrow_u64(0, rax, WC_L64(r8, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -184)); + WC_S64(rdi, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -176)); + WC_S64(rdi, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -168)); + WC_S64(rdi, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -160)); + WC_S64(rdi, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -152)); + WC_S64(rdi, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -144)); + WC_S64(rdi, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -136)); + WC_S64(rdi, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -128)); + WC_S64(rdi, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -120)); + WC_S64(rdi, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -112)); + WC_S64(rdi, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -104)); + WC_S64(rdi, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -96)); + WC_S64(rdi, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -88)); + WC_S64(rdi, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -80)); + WC_S64(rdi, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -72)); + WC_S64(rdi, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -64)); + WC_S64(rdi, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -56)); + WC_S64(rdi, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -48)); + WC_S64(rdi, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -40)); + WC_S64(rdi, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -32)); + WC_S64(rdi, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -24)); + WC_S64(rdi, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -16)); + WC_S64(rdi, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -8)); + WC_S64(rdi, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 0)); + WC_S64(rdi, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 184), (unsigned long long*)&rdx); + WC_S64(rdi, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 384)); + rdi = (word64)(rdi + 0x240); + /* Add in word */ + rax = (word64)(WC_L64(rdi, 0)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 184) = (word64)(rdx); + rsi = (word64)(WC_L64(rsp, 392)); + rdi = (word64)(WC_L64(rsp, 384)); +} + +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * Karatsuba: ah^2, al^2, (al - ah)^2 + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_3072_sqr_avx2_48(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rcx, r8, r9, rdx, rax; + XALIGNED(32) WC_X64I_SLOT stk[52]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 416; + rsp = (word64)(rsp - 400); + WC_S64(rsp, 384) = (word64)(rdi); + WC_S64(rsp, 392) = (word64)(rsi); + rcx = (word64)(0); + r8 = (word64)(rsp); + r9 = (word64)(rsi + 192); + rdx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(r9, 0), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 8)); + WC_S64(r8, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 8), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 16)); + WC_S64(r8, 8) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 16), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r8, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 24), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 32)); + WC_S64(r8, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 32), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 40)); + WC_S64(r8, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 40), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 48)); + WC_S64(r8, 40) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 48), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 56)); + WC_S64(r8, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 56), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 64)); + WC_S64(r8, 56) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 64), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r8, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 72), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 80)); + WC_S64(r8, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 80), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 88)); + WC_S64(r8, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 88), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 96)); + WC_S64(r8, 88) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 96), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 104)); + WC_S64(r8, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 104), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 112)); + WC_S64(r8, 104) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 112), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r8, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 120), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 128)); + WC_S64(r8, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 128), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 136)); + WC_S64(r8, 128) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 136), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 144)); + WC_S64(r8, 136) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 144), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 152)); + WC_S64(r8, 144) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 152), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 160)); + WC_S64(r8, 152) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 160), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 168)); + WC_S64(r8, 160) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 168), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 176)); + WC_S64(r8, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 176), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 184)); + WC_S64(r8, 176) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 184), (unsigned long long*)&rax); + WC_S64(r8, 184) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* Cond Negate */ + rdx = (word64)(WC_L64(r8, 0)); + r9 = (word64)(rcx); + rdx = (word64)(rdx ^ rcx); + r9 = (word64)(0 - r9); + cf = _subborrow_u64(0, rdx, rcx, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 8)); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(r8, 0) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 16)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 8) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 24)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 16) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 32)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 24) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 40)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 32) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 48)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 40) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 56)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 48) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 64)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 56) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 72)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 64) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 80)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 72) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 88)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 80) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 96)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 88) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 104)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 96) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 112)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 104) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 120)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 112) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 128)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 120) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 136)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 128) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 144)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 136) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 152)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 144) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 160)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 152) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 168)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 160) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 176)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 168) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 184)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 176) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + rax = (word64)(rax + r9); + WC_S64(r8, 184) = (word64)(rax); + rsi = (word64)(r8); + rdi = (word64)(rsp); + (void)sp_3072_sqr_avx2_24((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); + rsi = (word64)(WC_L64(rsp, 392)); + rdi = (word64)(WC_L64(rsp, 384)); + rsi = (word64)(rsi + 0xc0); + rdi = (word64)(rdi + 0x180); + (void)sp_3072_sqr_avx2_24((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); + rsi = (word64)(WC_L64(rsp, 392)); + rdi = (word64)(WC_L64(rsp, 384)); + (void)sp_3072_sqr_avx2_24((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); +#ifdef _WIN64 + rsi = (word64)(WC_L64(rsp, 392)); + rdi = (word64)(WC_L64(rsp, 384)); +#endif /* _WIN64 */ + rsi = (word64)(WC_L64(rsp, 384)); + r8 = (word64)(rsp + 192); + rsi = (word64)(rsi + 0x240); + rcx = (word64)(0); + rax = (word64)(WC_L64(r8, -192)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -184)); + WC_S64(r8, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -176)); + WC_S64(r8, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -168)); + WC_S64(r8, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -160)); + WC_S64(r8, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -152)); + WC_S64(r8, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -144)); + WC_S64(r8, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -136)); + WC_S64(r8, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -128)); + WC_S64(r8, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 128)); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 136)); + WC_S64(r8, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 144)); + WC_S64(r8, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 152)); + WC_S64(r8, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 160)); + WC_S64(r8, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 168)); + WC_S64(r8, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 176)); + WC_S64(r8, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 184)); + WC_S64(r8, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 184), (unsigned long long*)&rdx); + WC_S64(r8, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rsi = (word64)(rsi - 0x180); + rax = (word64)(WC_L64(r8, -192)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -184)); + WC_S64(r8, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -176)); + WC_S64(r8, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -168)); + WC_S64(r8, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -160)); + WC_S64(r8, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -152)); + WC_S64(r8, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -144)); + WC_S64(r8, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -136)); + WC_S64(r8, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -128)); + WC_S64(r8, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 128)); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 136)); + WC_S64(r8, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 144)); + WC_S64(r8, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 152)); + WC_S64(r8, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 160)); + WC_S64(r8, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 168)); + WC_S64(r8, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 176)); + WC_S64(r8, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 184)); + WC_S64(r8, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 184), (unsigned long long*)&rdx); + WC_S64(r8, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 384)); + rcx = (word64)(0 - rcx); + rdi = (word64)(rdi + 0x180); + rax = (word64)(WC_L64(rdi, -192)); + cf = _subborrow_u64(0, rax, WC_L64(r8, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -184)); + WC_S64(rdi, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -176)); + WC_S64(rdi, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -168)); + WC_S64(rdi, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -160)); + WC_S64(rdi, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -152)); + WC_S64(rdi, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -144)); + WC_S64(rdi, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -136)); + WC_S64(rdi, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -128)); + WC_S64(rdi, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -120)); + WC_S64(rdi, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -112)); + WC_S64(rdi, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -104)); + WC_S64(rdi, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -96)); + WC_S64(rdi, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -88)); + WC_S64(rdi, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -80)); + WC_S64(rdi, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -72)); + WC_S64(rdi, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -64)); + WC_S64(rdi, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -56)); + WC_S64(rdi, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -48)); + WC_S64(rdi, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -40)); + WC_S64(rdi, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -32)); + WC_S64(rdi, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -24)); + WC_S64(rdi, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -16)); + WC_S64(rdi, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -8)); + WC_S64(rdi, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 0)); + WC_S64(rdi, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 184), (unsigned long long*)&rdx); + WC_S64(rdi, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 384)); + rdi = (word64)(rdi + 0x240); + /* Add in word */ + rax = (word64)(WC_L64(rdi, 0)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 184) = (word64)(rdx); + rsi = (word64)(WC_L64(rsp, 392)); + rdi = (word64)(WC_L64(rsp, 384)); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_3072_mul_d_48(sp_digit* r, const sp_digit* a, sp_digit b) +{ + word64 rdi, rsi, rcx, rax, r10, rdx = 0, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)b; + + /* A[0] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + WC_S64(rdi, 0) = (word64)(r8); + /* A[1] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 88) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 104) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 112) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[16] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 128)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[17] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 136)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 136) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[18] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 144)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 144) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[19] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 152)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 152) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[20] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 160)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 160) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[21] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 168)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[22] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 176)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[23] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 184)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 184) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[24] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 192)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 192) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[25] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 200)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 200) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[26] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 208)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 208) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[27] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 216)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 216) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[28] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 224)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 224) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[29] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 232)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 232) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[30] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 240)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 240) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[31] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 248)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 248) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[32] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 256)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 256) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[33] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 264)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 264) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[34] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 272)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 272) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[35] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 280)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 280) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[36] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 288)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 288) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[37] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 296)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 296) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[38] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 304)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 304) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[39] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 312)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 312) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[40] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 320)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 320) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[41] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 328)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 328) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[42] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 336)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 336) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[43] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 344)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 344) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[44] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 352)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 352) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[45] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 360)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 360) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[46] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 368)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 368) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[47] * B */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 376)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 376) = (word64)(r10); + WC_S64(rdi, 384) = (word64)(r8); +} + +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_3072_cond_sub_24(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, r8, r9, rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[24]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 192); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rdx, 136)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 128) = (word64)(r8); + WC_S64(rsp, 136) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 144)); + r9 = (word64)(WC_L64(rdx, 152)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 144) = (word64)(r8); + WC_S64(rsp, 152) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 160)); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 160) = (word64)(r8); + WC_S64(rsp, 168) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rdx, 184)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 176) = (word64)(r8); + WC_S64(rsp, 184) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _subborrow_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsp, 72)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 80)); + rdx = (word64)(WC_L64(rsp, 80)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsp, 88)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 96)); + rdx = (word64)(WC_L64(rsp, 96)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsp, 104)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 112)); + rdx = (word64)(WC_L64(rsp, 112)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsp, 120)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 128)); + rdx = (word64)(WC_L64(rsp, 128)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 136)); + rdx = (word64)(WC_L64(rsp, 136)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 144)); + rdx = (word64)(WC_L64(rsp, 144)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 136) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 152)); + rdx = (word64)(WC_L64(rsp, 152)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 144) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 160)); + rdx = (word64)(WC_L64(rsp, 160)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 152) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 168)); + rdx = (word64)(WC_L64(rsp, 168)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 160) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 176)); + rdx = (word64)(WC_L64(rsp, 176)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 184)); + rdx = (word64)(WC_L64(rsp, 184)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r8); + WC_S64(rdi, 184) = (word64)(r9); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Reduce the number back to 3072 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_3072_mont_reduce_24(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rsi, rcx, r15, r8, r13, r14, r11 = 0, rax = 0, r10 = 0, + rdx = 0, r9 = 0, r12 = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)mp; + + r15 = (word64)(0); + /* i = 24 */ + r8 = (word64)(0x18); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 8)); +L_3072_mont_reduce_24_loop: + /* mu = a[i] * mp */ + r11 = (word64)(r13); + r11 = (word64)(r11 * rcx); + /* a[i+0] += m[0] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* a[i+1] += m[1] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r13, r10, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+2] += m[2] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+3] += m[3] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 24) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+4] += m[4] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rdi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+5] += m[5] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 40) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+6] += m[6] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rdi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 48) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+7] += m[7] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r12 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 56) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+8] += m[8] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r12 = (word64)(WC_L64(rdi, 64)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 64) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+9] += m[9] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r12 = (word64)(WC_L64(rdi, 72)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 72) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+10] += m[10] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r12 = (word64)(WC_L64(rdi, 80)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 80) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+11] += m[11] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r12 = (word64)(WC_L64(rdi, 88)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 88) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+12] += m[12] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r12 = (word64)(WC_L64(rdi, 96)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 96) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+13] += m[13] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + r12 = (word64)(WC_L64(rdi, 104)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 104) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+14] += m[14] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r12 = (word64)(WC_L64(rdi, 112)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 112) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+15] += m[15] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + r12 = (word64)(WC_L64(rdi, 120)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 120) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+16] += m[16] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 128)); + r12 = (word64)(WC_L64(rdi, 128)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 128) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+17] += m[17] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 136)); + r12 = (word64)(WC_L64(rdi, 136)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 136) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+18] += m[18] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 144)); + r12 = (word64)(WC_L64(rdi, 144)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 144) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+19] += m[19] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 152)); + r12 = (word64)(WC_L64(rdi, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 152) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+20] += m[20] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 160)); + r12 = (word64)(WC_L64(rdi, 160)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 160) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+21] += m[21] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 168)); + r12 = (word64)(WC_L64(rdi, 168)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 168) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+22] += m[22] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 176)); + r12 = (word64)(WC_L64(rdi, 176)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 176) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+23] += m[23] * mu */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 184)); + r12 = (word64)(WC_L64(rdi, 184)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, r15, (unsigned long long*)&rdx); + r15 = (word64)(0); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 184) = (word64)(r12); + cf = _addcarry_u64(cf, WC_L64(rdi, 192), rdx, (unsigned long long*)&WC_S64( + rdi, 192)); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* i -= 1 */ + rdi = (word64)(rdi + 8); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_3072_mont_reduce_24_loop; + } + WC_S64(rdi, 0) = (word64)(r13); + WC_S64(rdi, 8) = (word64)(r14); + r15 = (word64)(0 - r15); +#ifdef _WIN64 + rdx = (word64)(rsi); + rcx = (word64)(r15); +#else + rcx = (word64)(r15); + rdx = (word64)(rsi); +#endif /* _WIN64 */ + rsi = (word64)(rdi); + rdi = (word64)(rdi); + rdi = (word64)(rdi - 0xc0); + (void)sp_3072_cond_sub_24((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx, (sp_digit)rcx); +} + +#ifdef HAVE_INTEL_AVX2 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_3072_cond_sub_avx2_24(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r10, r8, r9, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _subborrow_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 64) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 80) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 88) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 104)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 96) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 104) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 112) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rsi, 128)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 128) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 144)); + r8 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 136) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 152)); + r9 = (word64)(WC_L64(rsi, 152)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 144) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 152) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 160) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rsi, 176)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 176) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + WC_S64(rdi, 184) = (word64)(r10); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_3072_mul_d_24(sp_digit* r, const sp_digit* a, sp_digit b) +{ + word64 rdi, rsi, rcx, rax, r10, rdx = 0, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)b; + + /* A[0] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + WC_S64(rdi, 0) = (word64)(r8); + /* A[1] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 88) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 104) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 112) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[16] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 128)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[17] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 136)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 136) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[18] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 144)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 144) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[19] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 152)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 152) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[20] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 160)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 160) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[21] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 168)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[22] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 176)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[23] * B */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 184)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 184) = (word64)(r10); + WC_S64(rdi, 192) = (word64)(r8); +} + +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_3072_mul_d_avx2_24(sp_digit* r, const sp_digit* a, + const sp_digit b) +{ + sp_3072_mul_d_24(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef _WIN64 +/* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) + * + * @param [in] d1 The high order half of the number to divide. + * @param [in] d0 The low order half of the number to divide. + * @param [in] div The dividend. + * + * @return The result of the division. + */ +WOLFSSL_LOCAL sp_digit div_3072_word_asm_24(sp_digit d1, sp_digit d0, + sp_digit div) +{ + word64 rdi, rsi, rcx, rax, rdx; + + rdi = (word64)(word64)d1; + rsi = (word64)(word64)d0; + rcx = (word64)(word64)div; + + rax = (word64)(rsi); + rdx = (word64)(rdi); + WC_X64I_DIV128(rax, rdx, rdx, rax, rcx); + return (sp_digit)(word64)rax; +} + +#endif /* _WIN64 */ +/* Compare a with b in constant time. + * + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + * + * @return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +WOLFSSL_LOCAL sp_int64 sp_3072_cmp_24(const sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + word64 zf7; + word64 zf8; + word64 zf9; + word64 zf10; + word64 zf11; + word64 zf12; + word64 zf13; + word64 zf14; + word64 zf15; + word64 zf16; + word64 zf17; + word64 zf18; + word64 zf19; + word64 zf20; + word64 zf21; + word64 zf22; + word64 zf23; + word64 zf24; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rcx = (word64)(0); + rdx = (word64)(-1); + rax = (word64)(-1); + r8 = (word64)(1); + r9 = (word64)(WC_L64(rdi, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf1 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf1) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 176)); + r10 = (word64)(WC_L64(rsi, 176)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf2 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf2) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 168)); + r10 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf3 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf3) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf4 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf4) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 152)); + r10 = (word64)(WC_L64(rsi, 152)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf5 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf5) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 144)); + r10 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf6 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf6) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf7 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf7) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 128)); + r10 = (word64)(WC_L64(rsi, 128)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf8 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf8) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 120)); + r10 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf9 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf9) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf10 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf10) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 104)); + r10 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf11 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf11) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf12 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf12) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf13 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf13) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf14 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf14) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf15 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf15) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf16 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf16) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(WC_L64(rsi, 56)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf17 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf17) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf18 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf18) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf19 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf19) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf20 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf20) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf21 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf21) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf22 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf22) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf23 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf23) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf24 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf24) != (0) ? rcx : rdx; + rax = (word64)(rax ^ rdx); + return (sp_int64)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 3072 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_3072_mont_reduce_avx2_24(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + sp_3072_mont_reduce_24(a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifndef WC_NO_CACHE_RESISTANT +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_3072_get_from_table_avx2_24(sp_digit* r, sp_digit** table, + int idx) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y10, y11, y12, y13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y10 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rdx)); + y11 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rax)); + y13 = _mm256_setzero_si256(); + y10 = _mm256_permutevar8x32_epi32(y10, y13); + y11 = _mm256_permutevar8x32_epi32(y11, y13); + /* START: 0-15 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 16 */ + rcx = (word64)(WC_L64(rsi, 128)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 17 */ + rcx = (word64)(WC_L64(rsi, 136)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 18 */ + rcx = (word64)(WC_L64(rsi, 144)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 19 */ + rcx = (word64)(WC_L64(rsi, 152)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 20 */ + rcx = (word64)(WC_L64(rsi, 160)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 21 */ + rcx = (word64)(WC_L64(rsi, 168)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 22 */ + rcx = (word64)(WC_L64(rsi, 176)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 23 */ + rcx = (word64)(WC_L64(rsi, 184)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 24 */ + rcx = (word64)(WC_L64(rsi, 192)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 25 */ + rcx = (word64)(WC_L64(rsi, 200)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 26 */ + rcx = (word64)(WC_L64(rsi, 208)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 27 */ + rcx = (word64)(WC_L64(rsi, 216)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 28 */ + rcx = (word64)(WC_L64(rsi, 224)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 29 */ + rcx = (word64)(WC_L64(rsi, 232)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 30 */ + rcx = (word64)(WC_L64(rsi, 240)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 31 */ + rcx = (word64)(WC_L64(rsi, 248)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + rdi = (word64)(rdi + 0x80); + /* END: 0-15 */ + /* START: 16-23 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 16 */ + rcx = (word64)(WC_L64(rsi, 128)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 17 */ + rcx = (word64)(WC_L64(rsi, 136)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 18 */ + rcx = (word64)(WC_L64(rsi, 144)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 19 */ + rcx = (word64)(WC_L64(rsi, 152)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 20 */ + rcx = (word64)(WC_L64(rsi, 160)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 21 */ + rcx = (word64)(WC_L64(rsi, 168)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 22 */ + rcx = (word64)(WC_L64(rsi, 176)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 23 */ + rcx = (word64)(WC_L64(rsi, 184)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 24 */ + rcx = (word64)(WC_L64(rsi, 192)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 25 */ + rcx = (word64)(WC_L64(rsi, 200)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 26 */ + rcx = (word64)(WC_L64(rsi, 208)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 27 */ + rcx = (word64)(WC_L64(rsi, 216)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 28 */ + rcx = (word64)(WC_L64(rsi, 224)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 29 */ + rcx = (word64)(WC_L64(rsi, 232)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 30 */ + rcx = (word64)(WC_L64(rsi, 240)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 31 */ + rcx = (word64)(WC_L64(rsi, 248)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + /* END: 16-23 */ +} + +#endif /* !WC_NO_CACHE_RESISTANT */ +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_3072_cond_sub_48(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, r8, r9, rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[48]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 384; + rsp = (word64)(rsp - 384); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rdx, 136)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 128) = (word64)(r8); + WC_S64(rsp, 136) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 144)); + r9 = (word64)(WC_L64(rdx, 152)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 144) = (word64)(r8); + WC_S64(rsp, 152) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 160)); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 160) = (word64)(r8); + WC_S64(rsp, 168) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rdx, 184)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 176) = (word64)(r8); + WC_S64(rsp, 184) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 192)); + r9 = (word64)(WC_L64(rdx, 200)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 192) = (word64)(r8); + WC_S64(rsp, 200) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 208)); + r9 = (word64)(WC_L64(rdx, 216)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 208) = (word64)(r8); + WC_S64(rsp, 216) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 224)); + r9 = (word64)(WC_L64(rdx, 232)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 224) = (word64)(r8); + WC_S64(rsp, 232) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 240)); + r9 = (word64)(WC_L64(rdx, 248)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 240) = (word64)(r8); + WC_S64(rsp, 248) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 256)); + r9 = (word64)(WC_L64(rdx, 264)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 256) = (word64)(r8); + WC_S64(rsp, 264) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 272)); + r9 = (word64)(WC_L64(rdx, 280)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 272) = (word64)(r8); + WC_S64(rsp, 280) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 288)); + r9 = (word64)(WC_L64(rdx, 296)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 288) = (word64)(r8); + WC_S64(rsp, 296) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 304)); + r9 = (word64)(WC_L64(rdx, 312)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 304) = (word64)(r8); + WC_S64(rsp, 312) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 320)); + r9 = (word64)(WC_L64(rdx, 328)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 320) = (word64)(r8); + WC_S64(rsp, 328) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 336)); + r9 = (word64)(WC_L64(rdx, 344)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 336) = (word64)(r8); + WC_S64(rsp, 344) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 352)); + r9 = (word64)(WC_L64(rdx, 360)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 352) = (word64)(r8); + WC_S64(rsp, 360) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 368)); + r9 = (word64)(WC_L64(rdx, 376)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 368) = (word64)(r8); + WC_S64(rsp, 376) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _subborrow_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsp, 72)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 80)); + rdx = (word64)(WC_L64(rsp, 80)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsp, 88)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 96)); + rdx = (word64)(WC_L64(rsp, 96)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsp, 104)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 112)); + rdx = (word64)(WC_L64(rsp, 112)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsp, 120)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 128)); + rdx = (word64)(WC_L64(rsp, 128)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 136)); + rdx = (word64)(WC_L64(rsp, 136)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 144)); + rdx = (word64)(WC_L64(rsp, 144)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 136) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 152)); + rdx = (word64)(WC_L64(rsp, 152)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 144) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 160)); + rdx = (word64)(WC_L64(rsp, 160)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 152) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 168)); + rdx = (word64)(WC_L64(rsp, 168)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 160) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 176)); + rdx = (word64)(WC_L64(rsp, 176)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 184)); + rdx = (word64)(WC_L64(rsp, 184)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 192)); + rdx = (word64)(WC_L64(rsp, 192)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 184) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 200)); + rdx = (word64)(WC_L64(rsp, 200)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 192) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 208)); + rdx = (word64)(WC_L64(rsp, 208)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 200) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 216)); + rdx = (word64)(WC_L64(rsp, 216)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 208) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 224)); + rdx = (word64)(WC_L64(rsp, 224)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 216) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 232)); + rdx = (word64)(WC_L64(rsp, 232)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 224) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 240)); + rdx = (word64)(WC_L64(rsp, 240)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 232) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 248)); + rdx = (word64)(WC_L64(rsp, 248)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 240) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 256)); + rdx = (word64)(WC_L64(rsp, 256)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 248) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 264)); + rdx = (word64)(WC_L64(rsp, 264)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 256) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 272)); + rdx = (word64)(WC_L64(rsp, 272)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 264) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 280)); + rdx = (word64)(WC_L64(rsp, 280)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 272) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 288)); + rdx = (word64)(WC_L64(rsp, 288)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 280) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 296)); + rdx = (word64)(WC_L64(rsp, 296)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 288) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 304)); + rdx = (word64)(WC_L64(rsp, 304)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 296) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 312)); + rdx = (word64)(WC_L64(rsp, 312)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 304) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 320)); + rdx = (word64)(WC_L64(rsp, 320)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 312) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 328)); + rdx = (word64)(WC_L64(rsp, 328)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 320) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 336)); + rdx = (word64)(WC_L64(rsp, 336)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 328) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 344)); + rdx = (word64)(WC_L64(rsp, 344)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 336) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 352)); + rdx = (word64)(WC_L64(rsp, 352)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 344) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 360)); + rdx = (word64)(WC_L64(rsp, 360)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 352) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 368)); + rdx = (word64)(WC_L64(rsp, 368)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 360) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 376)); + rdx = (word64)(WC_L64(rsp, 376)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 368) = (word64)(r8); + WC_S64(rdi, 376) = (word64)(r9); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Reduce the number back to 3072 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_3072_mont_reduce_48(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rsi, rcx, r15, r8, r13, r14, r11 = 0, rax = 0, r10 = 0, + rdx = 0, r9 = 0, r12 = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)mp; + + r15 = (word64)(0); + /* i = 48 */ + r8 = (word64)(0x30); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 8)); +L_3072_mont_reduce_48_loop: + /* mu = a[i] * mp */ + r11 = (word64)(r13); + r11 = (word64)(r11 * rcx); + /* a[i+0] += m[0] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* a[i+1] += m[1] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r13, r10, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+2] += m[2] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+3] += m[3] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 24) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+4] += m[4] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rdi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+5] += m[5] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 40) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+6] += m[6] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rdi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 48) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+7] += m[7] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r12 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 56) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+8] += m[8] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r12 = (word64)(WC_L64(rdi, 64)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 64) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+9] += m[9] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r12 = (word64)(WC_L64(rdi, 72)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 72) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+10] += m[10] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r12 = (word64)(WC_L64(rdi, 80)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 80) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+11] += m[11] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r12 = (word64)(WC_L64(rdi, 88)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 88) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+12] += m[12] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r12 = (word64)(WC_L64(rdi, 96)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 96) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+13] += m[13] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + r12 = (word64)(WC_L64(rdi, 104)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 104) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+14] += m[14] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r12 = (word64)(WC_L64(rdi, 112)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 112) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+15] += m[15] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + r12 = (word64)(WC_L64(rdi, 120)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 120) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+16] += m[16] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 128)); + r12 = (word64)(WC_L64(rdi, 128)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 128) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+17] += m[17] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 136)); + r12 = (word64)(WC_L64(rdi, 136)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 136) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+18] += m[18] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 144)); + r12 = (word64)(WC_L64(rdi, 144)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 144) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+19] += m[19] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 152)); + r12 = (word64)(WC_L64(rdi, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 152) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+20] += m[20] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 160)); + r12 = (word64)(WC_L64(rdi, 160)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 160) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+21] += m[21] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 168)); + r12 = (word64)(WC_L64(rdi, 168)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 168) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+22] += m[22] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 176)); + r12 = (word64)(WC_L64(rdi, 176)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 176) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+23] += m[23] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 184)); + r12 = (word64)(WC_L64(rdi, 184)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 184) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+24] += m[24] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 192)); + r12 = (word64)(WC_L64(rdi, 192)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 192) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+25] += m[25] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 200)); + r12 = (word64)(WC_L64(rdi, 200)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 200) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+26] += m[26] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 208)); + r12 = (word64)(WC_L64(rdi, 208)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 208) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+27] += m[27] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 216)); + r12 = (word64)(WC_L64(rdi, 216)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 216) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+28] += m[28] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 224)); + r12 = (word64)(WC_L64(rdi, 224)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 224) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+29] += m[29] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 232)); + r12 = (word64)(WC_L64(rdi, 232)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 232) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+30] += m[30] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 240)); + r12 = (word64)(WC_L64(rdi, 240)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 240) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+31] += m[31] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 248)); + r12 = (word64)(WC_L64(rdi, 248)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 248) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+32] += m[32] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 256)); + r12 = (word64)(WC_L64(rdi, 256)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 256) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+33] += m[33] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 264)); + r12 = (word64)(WC_L64(rdi, 264)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 264) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+34] += m[34] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 272)); + r12 = (word64)(WC_L64(rdi, 272)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 272) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+35] += m[35] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 280)); + r12 = (word64)(WC_L64(rdi, 280)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 280) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+36] += m[36] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 288)); + r12 = (word64)(WC_L64(rdi, 288)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 288) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+37] += m[37] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 296)); + r12 = (word64)(WC_L64(rdi, 296)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 296) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+38] += m[38] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 304)); + r12 = (word64)(WC_L64(rdi, 304)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 304) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+39] += m[39] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 312)); + r12 = (word64)(WC_L64(rdi, 312)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 312) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+40] += m[40] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 320)); + r12 = (word64)(WC_L64(rdi, 320)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 320) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+41] += m[41] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 328)); + r12 = (word64)(WC_L64(rdi, 328)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 328) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+42] += m[42] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 336)); + r12 = (word64)(WC_L64(rdi, 336)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 336) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+43] += m[43] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 344)); + r12 = (word64)(WC_L64(rdi, 344)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 344) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+44] += m[44] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 352)); + r12 = (word64)(WC_L64(rdi, 352)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 352) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+45] += m[45] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 360)); + r12 = (word64)(WC_L64(rdi, 360)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 360) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+46] += m[46] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 368)); + r12 = (word64)(WC_L64(rdi, 368)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 368) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+47] += m[47] * mu */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 376)); + r12 = (word64)(WC_L64(rdi, 376)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, r15, (unsigned long long*)&rdx); + r15 = (word64)(0); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 376) = (word64)(r12); + cf = _addcarry_u64(cf, WC_L64(rdi, 384), rdx, (unsigned long long*)&WC_S64( + rdi, 384)); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* i -= 1 */ + rdi = (word64)(rdi + 8); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_3072_mont_reduce_48_loop; + } + WC_S64(rdi, 0) = (word64)(r13); + WC_S64(rdi, 8) = (word64)(r14); + r15 = (word64)(0 - r15); +#ifdef _WIN64 + rdx = (word64)(rsi); + rcx = (word64)(r15); +#else + rcx = (word64)(r15); + rdx = (word64)(rsi); +#endif /* _WIN64 */ + rsi = (word64)(rdi); + rdi = (word64)(rdi); + rdi = (word64)(rdi - 0x180); + (void)sp_3072_cond_sub_48((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx, (sp_digit)rcx); +} + +/* Sub b from a into r. (r = a - b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_3072_sub_48(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, r8, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rcx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 96)); + WC_S64(rdi, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 96), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 104)); + WC_S64(rdi, 96) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 112)); + WC_S64(rdi, 104) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 112), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 120)); + WC_S64(rdi, 112) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 120), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(rdi, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 136), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 144)); + WC_S64(rdi, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 144), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 152)); + WC_S64(rdi, 144) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 152), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 160)); + WC_S64(rdi, 152) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 160), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 168)); + WC_S64(rdi, 160) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 168), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(rdi, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 184), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 192)); + WC_S64(rdi, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 192), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 200)); + WC_S64(rdi, 192) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 200), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 208)); + WC_S64(rdi, 200) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 208), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 216)); + WC_S64(rdi, 208) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 216), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 224)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 232)); + WC_S64(rdi, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 232), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 240)); + WC_S64(rdi, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 240), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 248)); + WC_S64(rdi, 240) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 248), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 256)); + WC_S64(rdi, 248) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 256), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 264)); + WC_S64(rdi, 256) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 264), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 272)); + WC_S64(rdi, 264) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 280)); + WC_S64(rdi, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 280), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 288)); + WC_S64(rdi, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 288), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 296)); + WC_S64(rdi, 288) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 296), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 304)); + WC_S64(rdi, 296) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 304), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 312)); + WC_S64(rdi, 304) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 312), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 320)); + WC_S64(rdi, 312) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 328)); + WC_S64(rdi, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 328), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 336)); + WC_S64(rdi, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 336), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 344)); + WC_S64(rdi, 336) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 344), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 352)); + WC_S64(rdi, 344) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 352), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 360)); + WC_S64(rdi, 352) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 360), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 368)); + WC_S64(rdi, 360) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 376)); + WC_S64(rdi, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 376), (unsigned long long*)&r8); + WC_S64(rdi, 376) = (word64)(r8); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_3072_mul_d_avx2_48(sp_digit* r, const sp_digit* a, + const sp_digit b) +{ + sp_3072_mul_d_48(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef _WIN64 +/* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) + * + * @param [in] d1 The high order half of the number to divide. + * @param [in] d0 The low order half of the number to divide. + * @param [in] div The dividend. + * + * @return The result of the division. + */ +WOLFSSL_LOCAL sp_digit div_3072_word_asm_48(sp_digit d1, sp_digit d0, + sp_digit div) +{ + word64 rdi, rsi, rcx, rax, rdx; + + rdi = (word64)(word64)d1; + rsi = (word64)(word64)d0; + rcx = (word64)(word64)div; + + rax = (word64)(rsi); + rdx = (word64)(rdi); + WC_X64I_DIV128(rax, rdx, rdx, rax, rcx); + return (sp_digit)(word64)rax; +} + +#endif /* _WIN64 */ +#ifdef HAVE_INTEL_AVX2 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_3072_cond_sub_avx2_48(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r10, r8, r9, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _subborrow_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 64) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 80) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 88) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 104)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 96) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 104) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 112) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rsi, 128)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 128) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 144)); + r8 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 136) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 152)); + r9 = (word64)(WC_L64(rsi, 152)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 144) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 152) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 160) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rsi, 176)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 176) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 192)); + r8 = (word64)(WC_L64(rsi, 192)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 184) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 200)); + r9 = (word64)(WC_L64(rsi, 200)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 192) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 208)); + r10 = (word64)(WC_L64(rsi, 208)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 200) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 216)); + r8 = (word64)(WC_L64(rsi, 216)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 208) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 224)); + r9 = (word64)(WC_L64(rsi, 224)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 232)); + r10 = (word64)(WC_L64(rsi, 232)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 224) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 240)); + r8 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 232) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 248)); + r9 = (word64)(WC_L64(rsi, 248)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 240) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 256)); + r10 = (word64)(WC_L64(rsi, 256)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 248) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 264)); + r8 = (word64)(WC_L64(rsi, 264)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 256) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 272)); + r9 = (word64)(WC_L64(rsi, 272)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 264) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 280)); + r10 = (word64)(WC_L64(rsi, 280)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 272) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 288)); + r8 = (word64)(WC_L64(rsi, 288)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 280) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 296)); + r9 = (word64)(WC_L64(rsi, 296)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 288) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 304)); + r10 = (word64)(WC_L64(rsi, 304)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 296) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 312)); + r8 = (word64)(WC_L64(rsi, 312)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 304) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 320)); + r9 = (word64)(WC_L64(rsi, 320)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 312) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 328)); + r10 = (word64)(WC_L64(rsi, 328)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 320) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 336)); + r8 = (word64)(WC_L64(rsi, 336)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 328) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 344)); + r9 = (word64)(WC_L64(rsi, 344)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 336) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 352)); + r10 = (word64)(WC_L64(rsi, 352)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 344) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 360)); + r8 = (word64)(WC_L64(rsi, 360)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 352) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 368)); + r9 = (word64)(WC_L64(rsi, 368)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 360) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 376)); + r10 = (word64)(WC_L64(rsi, 376)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 368) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + WC_S64(rdi, 376) = (word64)(r10); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Compare a with b in constant time. + * + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + * + * @return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +WOLFSSL_LOCAL sp_int64 sp_3072_cmp_48(const sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + word64 zf7; + word64 zf8; + word64 zf9; + word64 zf10; + word64 zf11; + word64 zf12; + word64 zf13; + word64 zf14; + word64 zf15; + word64 zf16; + word64 zf17; + word64 zf18; + word64 zf19; + word64 zf20; + word64 zf21; + word64 zf22; + word64 zf23; + word64 zf24; + word64 zf25; + word64 zf26; + word64 zf27; + word64 zf28; + word64 zf29; + word64 zf30; + word64 zf31; + word64 zf32; + word64 zf33; + word64 zf34; + word64 zf35; + word64 zf36; + word64 zf37; + word64 zf38; + word64 zf39; + word64 zf40; + word64 zf41; + word64 zf42; + word64 zf43; + word64 zf44; + word64 zf45; + word64 zf46; + word64 zf47; + word64 zf48; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rcx = (word64)(0); + rdx = (word64)(-1); + rax = (word64)(-1); + r8 = (word64)(1); + r9 = (word64)(WC_L64(rdi, 376)); + r10 = (word64)(WC_L64(rsi, 376)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf1 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf1) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 368)); + r10 = (word64)(WC_L64(rsi, 368)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf2 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf2) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 360)); + r10 = (word64)(WC_L64(rsi, 360)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf3 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf3) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 352)); + r10 = (word64)(WC_L64(rsi, 352)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf4 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf4) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 344)); + r10 = (word64)(WC_L64(rsi, 344)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf5 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf5) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 336)); + r10 = (word64)(WC_L64(rsi, 336)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf6 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf6) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 328)); + r10 = (word64)(WC_L64(rsi, 328)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf7 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf7) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 320)); + r10 = (word64)(WC_L64(rsi, 320)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf8 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf8) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 312)); + r10 = (word64)(WC_L64(rsi, 312)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf9 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf9) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 304)); + r10 = (word64)(WC_L64(rsi, 304)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf10 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf10) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 296)); + r10 = (word64)(WC_L64(rsi, 296)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf11 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf11) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 288)); + r10 = (word64)(WC_L64(rsi, 288)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf12 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf12) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 280)); + r10 = (word64)(WC_L64(rsi, 280)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf13 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf13) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 272)); + r10 = (word64)(WC_L64(rsi, 272)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf14 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf14) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 264)); + r10 = (word64)(WC_L64(rsi, 264)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf15 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf15) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 256)); + r10 = (word64)(WC_L64(rsi, 256)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf16 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf16) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 248)); + r10 = (word64)(WC_L64(rsi, 248)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf17 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf17) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 240)); + r10 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf18 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf18) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 232)); + r10 = (word64)(WC_L64(rsi, 232)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf19 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf19) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 224)); + r10 = (word64)(WC_L64(rsi, 224)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf20 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf20) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 216)); + r10 = (word64)(WC_L64(rsi, 216)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf21 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf21) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 208)); + r10 = (word64)(WC_L64(rsi, 208)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf22 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf22) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 200)); + r10 = (word64)(WC_L64(rsi, 200)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf23 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf23) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 192)); + r10 = (word64)(WC_L64(rsi, 192)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf24 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf24) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf25 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf25) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 176)); + r10 = (word64)(WC_L64(rsi, 176)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf26 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf26) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 168)); + r10 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf27 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf27) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf28 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf28) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 152)); + r10 = (word64)(WC_L64(rsi, 152)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf29 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf29) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 144)); + r10 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf30 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf30) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf31 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf31) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 128)); + r10 = (word64)(WC_L64(rsi, 128)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf32 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf32) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 120)); + r10 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf33 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf33) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf34 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf34) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 104)); + r10 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf35 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf35) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf36 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf36) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf37 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf37) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf38 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf38) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf39 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf39) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf40 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf40) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(WC_L64(rsi, 56)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf41 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf41) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf42 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf42) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf43 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf43) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf44 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf44) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf45 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf45) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf46 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf46) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf47 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf47) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf48 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf48) != (0) ? rcx : rdx; + rax = (word64)(rax ^ rdx); + return (sp_int64)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 3072 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_3072_mont_reduce_avx2_48(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + sp_3072_mont_reduce_48(a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifndef WC_NO_CACHE_RESISTANT +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_3072_get_from_table_avx2_48(sp_digit* r, sp_digit** table, + int idx) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y10, y11, y12, y13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y10 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rdx)); + y11 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rax)); + y13 = _mm256_setzero_si256(); + y10 = _mm256_permutevar8x32_epi32(y10, y13); + y11 = _mm256_permutevar8x32_epi32(y11, y13); + /* START: 0-15 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + rdi = (word64)(rdi + 0x80); + /* END: 0-15 */ + /* START: 16-31 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + rdi = (word64)(rdi + 0x80); + /* END: 16-31 */ + /* START: 32-47 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + /* END: 32-47 */ +} + +#endif /* !WC_NO_CACHE_RESISTANT */ +/* Conditionally add a and b using the mask m. + * m is -1 to add and 0 when not. + * + * @param [out] r A single precision number representing conditional add + * result. + * @param [in] a A single precision number to add with. + * @param [in] b A single precision number to add. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_3072_cond_add_24(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, rax, r8, r9; + XALIGNED(32) WC_X64I_SLOT stk[24]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 192; + rsp = (word64)(rsp - 192); + rax = (word64)(0); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rdx, 136)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 128) = (word64)(r8); + WC_S64(rsp, 136) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 144)); + r9 = (word64)(WC_L64(rdx, 152)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 144) = (word64)(r8); + WC_S64(rsp, 152) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 160)); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 160) = (word64)(r8); + WC_S64(rsp, 168) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rdx, 184)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 176) = (word64)(r8); + WC_S64(rsp, 184) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _addcarry_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsp, 72)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 80)); + rdx = (word64)(WC_L64(rsp, 80)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsp, 88)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 96)); + rdx = (word64)(WC_L64(rsp, 96)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsp, 104)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 112)); + rdx = (word64)(WC_L64(rsp, 112)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsp, 120)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 128)); + rdx = (word64)(WC_L64(rsp, 128)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 136)); + rdx = (word64)(WC_L64(rsp, 136)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 144)); + rdx = (word64)(WC_L64(rsp, 144)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 136) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 152)); + rdx = (word64)(WC_L64(rsp, 152)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 144) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 160)); + rdx = (word64)(WC_L64(rsp, 160)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 152) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 168)); + rdx = (word64)(WC_L64(rsp, 168)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 160) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 176)); + rdx = (word64)(WC_L64(rsp, 176)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 184)); + rdx = (word64)(WC_L64(rsp, 184)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r8); + WC_S64(rdi, 184) = (word64)(r9); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Conditionally add a and b using the mask m. + * m is -1 to add and 0 when not. + * + * @param [out] r A single precision number representing conditional add + * result. + * @param [in] a A single precision number to add with. + * @param [in] b A single precision number to add. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_3072_cond_add_avx2_24(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rax = (word64)(0); + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 64) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 80) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 88) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 104)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 96) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 104) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 112) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rsi, 128)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 128) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 144)); + r8 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 136) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 152)); + r9 = (word64)(WC_L64(rsi, 152)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 144) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 152) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 160) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rsi, 176)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 176) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + WC_S64(rdi, 184) = (word64)(r10); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Shift number left by n bit. (r = a << n) + * + * @param [out] r Result of left shift by n. + * @param [in] a Number to shift. + * @param [in] n Amoutnt o shift. + */ +WOLFSSL_LOCAL void sp_3072_lshift_48(sp_digit* r, const sp_digit* a, int n) +{ + word64 rdi, rsi, rcx, r10, r11, rdx, rax, r8, r9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r10 = (word64)(0); + r11 = (word64)(WC_L64(rsi, 344)); + rdx = (word64)(WC_L64(rsi, 352)); + rax = (word64)(WC_L64(rsi, 360)); + r8 = (word64)(WC_L64(rsi, 368)); + r9 = (word64)(WC_L64(rsi, 376)); + r10 = (word64)((r10 << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 352) = (word64)(rdx); + WC_S64(rdi, 360) = (word64)(rax); + WC_S64(rdi, 368) = (word64)(r8); + WC_S64(rdi, 376) = (word64)(r9); + WC_S64(rdi, 384) = (word64)(r10); + r9 = (word64)(WC_L64(rsi, 312)); + rdx = (word64)(WC_L64(rsi, 320)); + rax = (word64)(WC_L64(rsi, 328)); + r8 = (word64)(WC_L64(rsi, 336)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 320) = (word64)(rdx); + WC_S64(rdi, 328) = (word64)(rax); + WC_S64(rdi, 336) = (word64)(r8); + WC_S64(rdi, 344) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 280)); + rdx = (word64)(WC_L64(rsi, 288)); + rax = (word64)(WC_L64(rsi, 296)); + r8 = (word64)(WC_L64(rsi, 304)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 288) = (word64)(rdx); + WC_S64(rdi, 296) = (word64)(rax); + WC_S64(rdi, 304) = (word64)(r8); + WC_S64(rdi, 312) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 248)); + rdx = (word64)(WC_L64(rsi, 256)); + rax = (word64)(WC_L64(rsi, 264)); + r8 = (word64)(WC_L64(rsi, 272)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 256) = (word64)(rdx); + WC_S64(rdi, 264) = (word64)(rax); + WC_S64(rdi, 272) = (word64)(r8); + WC_S64(rdi, 280) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 216)); + rdx = (word64)(WC_L64(rsi, 224)); + rax = (word64)(WC_L64(rsi, 232)); + r8 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 224) = (word64)(rdx); + WC_S64(rdi, 232) = (word64)(rax); + WC_S64(rdi, 240) = (word64)(r8); + WC_S64(rdi, 248) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 184)); + rdx = (word64)(WC_L64(rsi, 192)); + rax = (word64)(WC_L64(rsi, 200)); + r8 = (word64)(WC_L64(rsi, 208)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 192) = (word64)(rdx); + WC_S64(rdi, 200) = (word64)(rax); + WC_S64(rdi, 208) = (word64)(r8); + WC_S64(rdi, 216) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 152)); + rdx = (word64)(WC_L64(rsi, 160)); + rax = (word64)(WC_L64(rsi, 168)); + r8 = (word64)(WC_L64(rsi, 176)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 160) = (word64)(rdx); + WC_S64(rdi, 168) = (word64)(rax); + WC_S64(rdi, 176) = (word64)(r8); + WC_S64(rdi, 184) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsi, 128)); + rax = (word64)(WC_L64(rsi, 136)); + r8 = (word64)(WC_L64(rsi, 144)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 128) = (word64)(rdx); + WC_S64(rdi, 136) = (word64)(rax); + WC_S64(rdi, 144) = (word64)(r8); + WC_S64(rdi, 152) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsi, 96)); + rax = (word64)(WC_L64(rsi, 104)); + r8 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 96) = (word64)(rdx); + WC_S64(rdi, 104) = (word64)(rax); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsi, 64)); + rax = (word64)(WC_L64(rsi, 72)); + r8 = (word64)(WC_L64(rsi, 80)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 64) = (word64)(rdx); + WC_S64(rdi, 72) = (word64)(rax); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsi, 32)); + rax = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 32) = (word64)(rdx); + WC_S64(rdi, 40) = (word64)(rax); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)(rdx << ((byte)rcx & 63)); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r11); +} + +#endif /* !WOLFSSL_SP_NO_3072 */ +#endif /* !WOLFSSL_SP_NO_3072 */ +#ifdef WOLFSSL_SP_4096 +#ifdef WOLFSSL_SP_4096 +/* Read big endian unsigned byte array into r. + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WOLFSSL_LOCAL void sp_4096_from_bin_bswap(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, r11, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x200); + r11 = (word64)(0); + goto L_4096_from_bin_bswap_64_end; +L_4096_from_bin_bswap_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_L64(r9, 56)); + r8 = (word64)(WC_L64(r9, 48)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_L64(r9, 40)); + r8 = (word64)(WC_L64(r9, 32)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(r9, 24)); + r8 = (word64)(WC_L64(r9, 16)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_L64(r9, 8)); + r8 = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_4096_from_bin_bswap_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_4096_from_bin_bswap_64_start; + } + goto L_4096_from_bin_bswap_8_end; +L_4096_from_bin_bswap_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_4096_from_bin_bswap_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_4096_from_bin_bswap_8_start; + } + if ((rcx) == (r11)) { + goto L_4096_from_bin_bswap_hi_end; + } + r8 = (word64)(r11); + rax = (word64)(r11); +L_4096_from_bin_bswap_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_4096_from_bin_bswap_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_4096_from_bin_bswap_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_4096_from_bin_bswap_zero_end; + } +L_4096_from_bin_bswap_zero_start: + WC_S64(rdi, 0) = (word64)(r11); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_4096_from_bin_bswap_zero_start; + } +L_4096_from_bin_bswap_zero_end: + ; + (void)rsi; +} + +#ifndef NO_MOVBE_SUPPORT +/* Read big endian unsigned byte array into r. + * Uses the movbe instruction which is an optional instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_4096_from_bin_movbe(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x200); + goto L_4096_from_bin_movbe_64_end; +L_4096_from_bin_movbe_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 56))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 48))); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 40))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 32))); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 24))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 16))); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 8))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_4096_from_bin_movbe_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_4096_from_bin_movbe_64_start; + } + goto L_4096_from_bin_movbe_8_end; +L_4096_from_bin_movbe_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_4096_from_bin_movbe_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_4096_from_bin_movbe_8_start; + } + if ((rcx) == (0)) { + goto L_4096_from_bin_movbe_hi_end; + } + r8 = (word64)(0); + rax = (word64)(0); +L_4096_from_bin_movbe_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_4096_from_bin_movbe_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_4096_from_bin_movbe_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_4096_from_bin_movbe_zero_end; + } +L_4096_from_bin_movbe_zero_start: + WC_S64(rdi, 0) = (word64)(0); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_4096_from_bin_movbe_zero_start; + } +L_4096_from_bin_movbe_zero_end: + ; + (void)rsi; +} + +#endif /* !NO_MOVBE_SUPPORT */ +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 512 + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WOLFSSL_LOCAL void sp_4096_to_bin_bswap_64(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rdi, 504)); + rax = (word64)(WC_L64(rdi, 496)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 488)); + rax = (word64)(WC_L64(rdi, 480)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 472)); + rax = (word64)(WC_L64(rdi, 464)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 32) = (word64)(rdx); + WC_S64(rsi, 40) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 456)); + rax = (word64)(WC_L64(rdi, 448)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 48) = (word64)(rdx); + WC_S64(rsi, 56) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 440)); + rax = (word64)(WC_L64(rdi, 432)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 64) = (word64)(rdx); + WC_S64(rsi, 72) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 424)); + rax = (word64)(WC_L64(rdi, 416)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 80) = (word64)(rdx); + WC_S64(rsi, 88) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 408)); + rax = (word64)(WC_L64(rdi, 400)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 96) = (word64)(rdx); + WC_S64(rsi, 104) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 392)); + rax = (word64)(WC_L64(rdi, 384)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 112) = (word64)(rdx); + WC_S64(rsi, 120) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 376)); + rax = (word64)(WC_L64(rdi, 368)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 128) = (word64)(rdx); + WC_S64(rsi, 136) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 360)); + rax = (word64)(WC_L64(rdi, 352)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 144) = (word64)(rdx); + WC_S64(rsi, 152) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 344)); + rax = (word64)(WC_L64(rdi, 336)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 160) = (word64)(rdx); + WC_S64(rsi, 168) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 328)); + rax = (word64)(WC_L64(rdi, 320)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 176) = (word64)(rdx); + WC_S64(rsi, 184) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 312)); + rax = (word64)(WC_L64(rdi, 304)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 192) = (word64)(rdx); + WC_S64(rsi, 200) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 296)); + rax = (word64)(WC_L64(rdi, 288)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 208) = (word64)(rdx); + WC_S64(rsi, 216) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 280)); + rax = (word64)(WC_L64(rdi, 272)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 224) = (word64)(rdx); + WC_S64(rsi, 232) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 264)); + rax = (word64)(WC_L64(rdi, 256)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 240) = (word64)(rdx); + WC_S64(rsi, 248) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 248)); + rax = (word64)(WC_L64(rdi, 240)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 256) = (word64)(rdx); + WC_S64(rsi, 264) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 232)); + rax = (word64)(WC_L64(rdi, 224)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 272) = (word64)(rdx); + WC_S64(rsi, 280) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 216)); + rax = (word64)(WC_L64(rdi, 208)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 288) = (word64)(rdx); + WC_S64(rsi, 296) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 200)); + rax = (word64)(WC_L64(rdi, 192)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 304) = (word64)(rdx); + WC_S64(rsi, 312) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 184)); + rax = (word64)(WC_L64(rdi, 176)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 320) = (word64)(rdx); + WC_S64(rsi, 328) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 168)); + rax = (word64)(WC_L64(rdi, 160)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 336) = (word64)(rdx); + WC_S64(rsi, 344) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 152)); + rax = (word64)(WC_L64(rdi, 144)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 352) = (word64)(rdx); + WC_S64(rsi, 360) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 136)); + rax = (word64)(WC_L64(rdi, 128)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 368) = (word64)(rdx); + WC_S64(rsi, 376) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 120)); + rax = (word64)(WC_L64(rdi, 112)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 384) = (word64)(rdx); + WC_S64(rsi, 392) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 104)); + rax = (word64)(WC_L64(rdi, 96)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 400) = (word64)(rdx); + WC_S64(rsi, 408) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 88)); + rax = (word64)(WC_L64(rdi, 80)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 416) = (word64)(rdx); + WC_S64(rsi, 424) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 72)); + rax = (word64)(WC_L64(rdi, 64)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 432) = (word64)(rdx); + WC_S64(rsi, 440) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 56)); + rax = (word64)(WC_L64(rdi, 48)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 448) = (word64)(rdx); + WC_S64(rsi, 456) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 40)); + rax = (word64)(WC_L64(rdi, 32)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 464) = (word64)(rdx); + WC_S64(rsi, 472) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 24)); + rax = (word64)(WC_L64(rdi, 16)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 480) = (word64)(rdx); + WC_S64(rsi, 488) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 8)); + rax = (word64)(WC_L64(rdi, 0)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 496) = (word64)(rdx); + WC_S64(rsi, 504) = (word64)(rax); +} + +#ifndef NO_MOVBE_SUPPORT +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 512 + * Uses the movbe instruction which is optional. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_4096_to_bin_movbe_64(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 504))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 496))); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 488))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 480))); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 472))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 464))); + WC_S64(rsi, 32) = (word64)(rdx); + WC_S64(rsi, 40) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 456))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 448))); + WC_S64(rsi, 48) = (word64)(rdx); + WC_S64(rsi, 56) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 440))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 432))); + WC_S64(rsi, 64) = (word64)(rdx); + WC_S64(rsi, 72) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 424))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 416))); + WC_S64(rsi, 80) = (word64)(rdx); + WC_S64(rsi, 88) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 408))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 400))); + WC_S64(rsi, 96) = (word64)(rdx); + WC_S64(rsi, 104) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 392))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 384))); + WC_S64(rsi, 112) = (word64)(rdx); + WC_S64(rsi, 120) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 376))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 368))); + WC_S64(rsi, 128) = (word64)(rdx); + WC_S64(rsi, 136) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 360))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 352))); + WC_S64(rsi, 144) = (word64)(rdx); + WC_S64(rsi, 152) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 344))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 336))); + WC_S64(rsi, 160) = (word64)(rdx); + WC_S64(rsi, 168) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 328))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 320))); + WC_S64(rsi, 176) = (word64)(rdx); + WC_S64(rsi, 184) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 312))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 304))); + WC_S64(rsi, 192) = (word64)(rdx); + WC_S64(rsi, 200) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 296))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 288))); + WC_S64(rsi, 208) = (word64)(rdx); + WC_S64(rsi, 216) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 280))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 272))); + WC_S64(rsi, 224) = (word64)(rdx); + WC_S64(rsi, 232) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 264))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 256))); + WC_S64(rsi, 240) = (word64)(rdx); + WC_S64(rsi, 248) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 248))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 240))); + WC_S64(rsi, 256) = (word64)(rdx); + WC_S64(rsi, 264) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 232))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 224))); + WC_S64(rsi, 272) = (word64)(rdx); + WC_S64(rsi, 280) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 216))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 208))); + WC_S64(rsi, 288) = (word64)(rdx); + WC_S64(rsi, 296) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 200))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 192))); + WC_S64(rsi, 304) = (word64)(rdx); + WC_S64(rsi, 312) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 184))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 176))); + WC_S64(rsi, 320) = (word64)(rdx); + WC_S64(rsi, 328) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 168))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 160))); + WC_S64(rsi, 336) = (word64)(rdx); + WC_S64(rsi, 344) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 152))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 144))); + WC_S64(rsi, 352) = (word64)(rdx); + WC_S64(rsi, 360) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 136))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 128))); + WC_S64(rsi, 368) = (word64)(rdx); + WC_S64(rsi, 376) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 120))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 112))); + WC_S64(rsi, 384) = (word64)(rdx); + WC_S64(rsi, 392) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 104))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 96))); + WC_S64(rsi, 400) = (word64)(rdx); + WC_S64(rsi, 408) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 88))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 80))); + WC_S64(rsi, 416) = (word64)(rdx); + WC_S64(rsi, 424) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 72))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 64))); + WC_S64(rsi, 432) = (word64)(rdx); + WC_S64(rsi, 440) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 56))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 48))); + WC_S64(rsi, 448) = (word64)(rdx); + WC_S64(rsi, 456) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 40))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 32))); + WC_S64(rsi, 464) = (word64)(rdx); + WC_S64(rsi, 472) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 24))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 16))); + WC_S64(rsi, 480) = (word64)(rdx); + WC_S64(rsi, 488) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 8))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 0))); + WC_S64(rsi, 496) = (word64)(rdx); + WC_S64(rsi, 504) = (word64)(rax); +} + +#endif /* NO_MOVBE_SUPPORT */ +/* Sub b from a into a. (a -= b) + * + * @param [in, out] a A single precision integer and result. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_4096_sub_in_place_64(sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rdi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 16), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 24), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 32), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 48), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 56), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 64), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 80), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 88), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 96), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 112), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 120), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 128), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 136), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 144), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 152), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 160), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 168), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 176), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 184), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 192)); + WC_S64(rdi, 184) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 192), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 200)); + WC_S64(rdi, 192) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 200), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 208)); + WC_S64(rdi, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 208), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 216)); + WC_S64(rdi, 208) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 216), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 224)); + WC_S64(rdi, 216) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 224), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 232)); + WC_S64(rdi, 224) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 232), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 240)); + WC_S64(rdi, 232) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 240), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 248)); + WC_S64(rdi, 240) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 248), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 256)); + WC_S64(rdi, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 256), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 264)); + WC_S64(rdi, 256) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 264), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 272)); + WC_S64(rdi, 264) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 272), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 280)); + WC_S64(rdi, 272) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 280), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 288)); + WC_S64(rdi, 280) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 288), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 296)); + WC_S64(rdi, 288) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 296), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 304)); + WC_S64(rdi, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 304), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 312)); + WC_S64(rdi, 304) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 312), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 320)); + WC_S64(rdi, 312) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 320), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 328)); + WC_S64(rdi, 320) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 328), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 336)); + WC_S64(rdi, 328) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 336), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 344)); + WC_S64(rdi, 336) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 344), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 352)); + WC_S64(rdi, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 352), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 360)); + WC_S64(rdi, 352) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 360), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 368)); + WC_S64(rdi, 360) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 368), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 376)); + WC_S64(rdi, 368) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 376), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 384)); + WC_S64(rdi, 376) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 384), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 392)); + WC_S64(rdi, 384) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 392), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 400)); + WC_S64(rdi, 392) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 400), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 408)); + WC_S64(rdi, 400) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 408), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 416)); + WC_S64(rdi, 408) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 416), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 424)); + WC_S64(rdi, 416) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 424), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 432)); + WC_S64(rdi, 424) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 432), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 440)); + WC_S64(rdi, 432) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 440), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 448)); + WC_S64(rdi, 440) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 448), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 456)); + WC_S64(rdi, 448) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 456), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 464)); + WC_S64(rdi, 456) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 464), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 472)); + WC_S64(rdi, 464) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 472), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 480)); + WC_S64(rdi, 472) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 480), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 488)); + WC_S64(rdi, 480) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 488), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 496)); + WC_S64(rdi, 488) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 496), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 504)); + WC_S64(rdi, 496) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 504), (unsigned long long*)&rcx); + WC_S64(rdi, 504) = (word64)(rcx); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_4096_add_64(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax, r8; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Add */ + rcx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(0); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 96)); + WC_S64(rdi, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 96), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 104)); + WC_S64(rdi, 96) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 112)); + WC_S64(rdi, 104) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 112), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 120)); + WC_S64(rdi, 112) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 120), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(rdi, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 136), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 144)); + WC_S64(rdi, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 144), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 152)); + WC_S64(rdi, 144) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 152), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 160)); + WC_S64(rdi, 152) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 160), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 168)); + WC_S64(rdi, 160) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 168), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(rdi, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 184), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 192)); + WC_S64(rdi, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 192), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 200)); + WC_S64(rdi, 192) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 200), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 208)); + WC_S64(rdi, 200) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 208), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 216)); + WC_S64(rdi, 208) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 216), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 224)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 232)); + WC_S64(rdi, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 232), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 240)); + WC_S64(rdi, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 240), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 248)); + WC_S64(rdi, 240) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 248), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 256)); + WC_S64(rdi, 248) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 256), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 264)); + WC_S64(rdi, 256) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 264), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 272)); + WC_S64(rdi, 264) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 280)); + WC_S64(rdi, 272) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 280), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 288)); + WC_S64(rdi, 280) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 288), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 296)); + WC_S64(rdi, 288) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 296), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 304)); + WC_S64(rdi, 296) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 304), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 312)); + WC_S64(rdi, 304) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 312), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 320)); + WC_S64(rdi, 312) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 328)); + WC_S64(rdi, 320) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 328), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 336)); + WC_S64(rdi, 328) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 336), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 344)); + WC_S64(rdi, 336) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 344), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 352)); + WC_S64(rdi, 344) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 352), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 360)); + WC_S64(rdi, 352) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 360), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 368)); + WC_S64(rdi, 360) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 376)); + WC_S64(rdi, 368) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 376), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 384)); + WC_S64(rdi, 376) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 384), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 392)); + WC_S64(rdi, 384) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 392), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 400)); + WC_S64(rdi, 392) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 400), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 408)); + WC_S64(rdi, 400) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 408), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 416)); + WC_S64(rdi, 408) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 416), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 424)); + WC_S64(rdi, 416) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 424), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 432)); + WC_S64(rdi, 424) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 432), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 440)); + WC_S64(rdi, 432) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 440), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 448)); + WC_S64(rdi, 440) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 448), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 456)); + WC_S64(rdi, 448) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 456), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 464)); + WC_S64(rdi, 456) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 464), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 472)); + WC_S64(rdi, 464) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 472), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 480)); + WC_S64(rdi, 472) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 480), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 488)); + WC_S64(rdi, 480) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 488), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 496)); + WC_S64(rdi, 488) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 496), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 504)); + WC_S64(rdi, 496) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 504), (unsigned long long*)&r8); + WC_S64(rdi, 504) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_4096_mul_64(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rsp, r10, r12, rax, r13, rcx, r8, r11, r14, r15, r9; + XALIGNED(32) WC_X64I_SLOT stk[200]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 1600; + rsp = (word64)(rsp - 1576); + WC_S64(rsp, 1536) = (word64)(rdi); + WC_S64(rsp, 1544) = (word64)(rsi); + WC_S64(rsp, 1552) = (word64)(rdx); + r10 = (word64)(rsp + 1024); + r12 = (word64)(rsi + 256); + /* Add */ + rax = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 248), (unsigned long long*)&rcx); + WC_S64(r10, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 1560) = (word64)(r13); + r11 = (word64)(rsp + 1280); + r12 = (word64)(rdx + 256); + /* Add */ + rax = (word64)(WC_L64(rdx, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 8)); + WC_S64(r11, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 16)); + WC_S64(r11, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 24)); + WC_S64(r11, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 32)); + WC_S64(r11, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 40)); + WC_S64(r11, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 48)); + WC_S64(r11, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 56)); + WC_S64(r11, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 64)); + WC_S64(r11, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 72)); + WC_S64(r11, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 80)); + WC_S64(r11, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 88)); + WC_S64(r11, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 96)); + WC_S64(r11, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 104)); + WC_S64(r11, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 112)); + WC_S64(r11, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 120)); + WC_S64(r11, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 128)); + WC_S64(r11, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 136)); + WC_S64(r11, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 144)); + WC_S64(r11, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 152)); + WC_S64(r11, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 160)); + WC_S64(r11, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 168)); + WC_S64(r11, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 176)); + WC_S64(r11, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 184)); + WC_S64(r11, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 192)); + WC_S64(r11, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 200)); + WC_S64(r11, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 208)); + WC_S64(r11, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 216)); + WC_S64(r11, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 224)); + WC_S64(r11, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 232)); + WC_S64(r11, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 240)); + WC_S64(r11, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 248)); + WC_S64(r11, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 248), (unsigned long long*)&rcx); + WC_S64(r11, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 1568) = (word64)(r14); + rdx = (word64)(r11); + rsi = (word64)(r10); + rdi = (word64)(rsp); + (void)sp_2048_mul_32((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 1552)); + rsi = (word64)(WC_L64(rsp, 1544)); + rdi = (word64)(rsp + 512); + rdx = (word64)(rdx + 0x100); + rsi = (word64)(rsi + 0x100); + (void)sp_2048_mul_32((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 1552)); + rsi = (word64)(WC_L64(rsp, 1544)); + rdi = (word64)(WC_L64(rsp, 1536)); + (void)sp_2048_mul_32((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi, ( + const sp_digit*)(size_t)rdx); +#ifdef _WIN64 + rdx = (word64)(WC_L64(rsp, 1552)); + rsi = (word64)(WC_L64(rsp, 1544)); + rdi = (word64)(WC_L64(rsp, 1536)); +#endif /* _WIN64 */ + r13 = (word64)(WC_L64(rsp, 1560)); + r14 = (word64)(WC_L64(rsp, 1568)); + r15 = (word64)(WC_L64(rsp, 1536)); + r9 = (word64)(r13); + r10 = (word64)(rsp + 1024); + r11 = (word64)(rsp + 1280); + r9 = (word64)(r9 & r14); + r13 = (word64)(0 - r13); + r14 = (word64)(0 - r14); + r15 = (word64)(r15 + 0x200); + rax = (word64)(WC_L64(r10, 0)); + rcx = (word64)(WC_L64(r11, 0)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 0) = (word64)(rax); + WC_S64(r11, 0) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 8)); + rcx = (word64)(WC_L64(r11, 8)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 8) = (word64)(rax); + WC_S64(r11, 8) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 16)); + rcx = (word64)(WC_L64(r11, 16)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 16) = (word64)(rax); + WC_S64(r11, 16) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 24)); + rcx = (word64)(WC_L64(r11, 24)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 24) = (word64)(rax); + WC_S64(r11, 24) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 32)); + rcx = (word64)(WC_L64(r11, 32)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 32) = (word64)(rax); + WC_S64(r11, 32) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 40)); + rcx = (word64)(WC_L64(r11, 40)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 40) = (word64)(rax); + WC_S64(r11, 40) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 48)); + rcx = (word64)(WC_L64(r11, 48)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 48) = (word64)(rax); + WC_S64(r11, 48) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 56)); + rcx = (word64)(WC_L64(r11, 56)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 56) = (word64)(rax); + WC_S64(r11, 56) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 64)); + rcx = (word64)(WC_L64(r11, 64)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 64) = (word64)(rax); + WC_S64(r11, 64) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 72)); + rcx = (word64)(WC_L64(r11, 72)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 72) = (word64)(rax); + WC_S64(r11, 72) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 80)); + rcx = (word64)(WC_L64(r11, 80)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 80) = (word64)(rax); + WC_S64(r11, 80) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 88)); + rcx = (word64)(WC_L64(r11, 88)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 88) = (word64)(rax); + WC_S64(r11, 88) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 96)); + rcx = (word64)(WC_L64(r11, 96)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 96) = (word64)(rax); + WC_S64(r11, 96) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 104)); + rcx = (word64)(WC_L64(r11, 104)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 104) = (word64)(rax); + WC_S64(r11, 104) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 112)); + rcx = (word64)(WC_L64(r11, 112)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 112) = (word64)(rax); + WC_S64(r11, 112) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 120)); + rcx = (word64)(WC_L64(r11, 120)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 120) = (word64)(rax); + WC_S64(r11, 120) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 128)); + rcx = (word64)(WC_L64(r11, 128)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 128) = (word64)(rax); + WC_S64(r11, 128) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 136)); + rcx = (word64)(WC_L64(r11, 136)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 136) = (word64)(rax); + WC_S64(r11, 136) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 144)); + rcx = (word64)(WC_L64(r11, 144)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 144) = (word64)(rax); + WC_S64(r11, 144) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 152)); + rcx = (word64)(WC_L64(r11, 152)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 152) = (word64)(rax); + WC_S64(r11, 152) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 160)); + rcx = (word64)(WC_L64(r11, 160)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 160) = (word64)(rax); + WC_S64(r11, 160) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 168)); + rcx = (word64)(WC_L64(r11, 168)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 168) = (word64)(rax); + WC_S64(r11, 168) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 176)); + rcx = (word64)(WC_L64(r11, 176)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 176) = (word64)(rax); + WC_S64(r11, 176) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 184)); + rcx = (word64)(WC_L64(r11, 184)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 184) = (word64)(rax); + WC_S64(r11, 184) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 192)); + rcx = (word64)(WC_L64(r11, 192)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 192) = (word64)(rax); + WC_S64(r11, 192) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 200)); + rcx = (word64)(WC_L64(r11, 200)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 200) = (word64)(rax); + WC_S64(r11, 200) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 208)); + rcx = (word64)(WC_L64(r11, 208)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 208) = (word64)(rax); + WC_S64(r11, 208) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 216)); + rcx = (word64)(WC_L64(r11, 216)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 216) = (word64)(rax); + WC_S64(r11, 216) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 224)); + rcx = (word64)(WC_L64(r11, 224)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 224) = (word64)(rax); + WC_S64(r11, 224) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 232)); + rcx = (word64)(WC_L64(r11, 232)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 232) = (word64)(rax); + WC_S64(r11, 232) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 240)); + rcx = (word64)(WC_L64(r11, 240)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 240) = (word64)(rax); + WC_S64(r11, 240) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 248)); + rcx = (word64)(WC_L64(r11, 248)); + rax = (word64)(rax & r14); + rcx = (word64)(rcx & r13); + WC_S64(r10, 248) = (word64)(rax); + WC_S64(r11, 248) = (word64)(rcx); + rax = (word64)(WC_L64(r10, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 248), (unsigned long long*)&rcx); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r11 = (word64)(rsp + 512); + r10 = (word64)(rsp); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 256)); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 264)); + WC_S64(r10, 256) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 272)); + WC_S64(r10, 264) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 280)); + WC_S64(r10, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 288)); + WC_S64(r10, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 296)); + WC_S64(r10, 288) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 304)); + WC_S64(r10, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 312)); + WC_S64(r10, 304) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 320)); + WC_S64(r10, 312) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 328)); + WC_S64(r10, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 336)); + WC_S64(r10, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 344)); + WC_S64(r10, 336) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 352)); + WC_S64(r10, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 360)); + WC_S64(r10, 352) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 368)); + WC_S64(r10, 360) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 376)); + WC_S64(r10, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 376), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 384)); + WC_S64(r10, 376) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 384), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 392)); + WC_S64(r10, 384) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 392), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 400)); + WC_S64(r10, 392) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 400), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 408)); + WC_S64(r10, 400) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 408), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 416)); + WC_S64(r10, 408) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 416), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 424)); + WC_S64(r10, 416) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 424), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 432)); + WC_S64(r10, 424) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 432), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 440)); + WC_S64(r10, 432) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 440), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 448)); + WC_S64(r10, 440) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 448), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 456)); + WC_S64(r10, 448) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 456), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 464)); + WC_S64(r10, 456) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 464), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 472)); + WC_S64(r10, 464) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 472), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 480)); + WC_S64(r10, 472) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 480), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 488)); + WC_S64(r10, 480) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 488), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 496)); + WC_S64(r10, 488) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 496), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 504)); + WC_S64(r10, 496) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 504), (unsigned long long*)&rax); + WC_S64(r10, 504) = (word64)(rax); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(rdi, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 256)); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 264)); + WC_S64(r10, 256) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 272)); + WC_S64(r10, 264) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 280)); + WC_S64(r10, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 288)); + WC_S64(r10, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 296)); + WC_S64(r10, 288) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 304)); + WC_S64(r10, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 312)); + WC_S64(r10, 304) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 320)); + WC_S64(r10, 312) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 328)); + WC_S64(r10, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 336)); + WC_S64(r10, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 344)); + WC_S64(r10, 336) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 352)); + WC_S64(r10, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 360)); + WC_S64(r10, 352) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 368)); + WC_S64(r10, 360) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 376)); + WC_S64(r10, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 376), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 384)); + WC_S64(r10, 376) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 384), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 392)); + WC_S64(r10, 384) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 392), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 400)); + WC_S64(r10, 392) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 400), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 408)); + WC_S64(r10, 400) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 408), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 416)); + WC_S64(r10, 408) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 416), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 424)); + WC_S64(r10, 416) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 424), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 432)); + WC_S64(r10, 424) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 432), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 440)); + WC_S64(r10, 432) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 440), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 448)); + WC_S64(r10, 440) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 448), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 456)); + WC_S64(r10, 448) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 456), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 464)); + WC_S64(r10, 456) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 464), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 472)); + WC_S64(r10, 464) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 472), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 480)); + WC_S64(r10, 472) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 480), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 488)); + WC_S64(r10, 480) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 488), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 496)); + WC_S64(r10, 488) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 496), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 504)); + WC_S64(r10, 496) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 504), (unsigned long long*)&rax); + WC_S64(r10, 504) = (word64)(rax); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + r15 = (word64)(r15 - 0x100); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r10, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 200)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 208)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 216)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 224)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 232)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 240)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 248)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 256)); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 264)); + WC_S64(r15, 256) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 272)); + WC_S64(r15, 264) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 280)); + WC_S64(r15, 272) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 288)); + WC_S64(r15, 280) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 296)); + WC_S64(r15, 288) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 304)); + WC_S64(r15, 296) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 312)); + WC_S64(r15, 304) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 320)); + WC_S64(r15, 312) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 328)); + WC_S64(r15, 320) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 336)); + WC_S64(r15, 328) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 344)); + WC_S64(r15, 336) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 352)); + WC_S64(r15, 344) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 360)); + WC_S64(r15, 352) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 368)); + WC_S64(r15, 360) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 376)); + WC_S64(r15, 368) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 376), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 384)); + WC_S64(r15, 376) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 384), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 392)); + WC_S64(r15, 384) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 392), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 400)); + WC_S64(r15, 392) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 400), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 408)); + WC_S64(r15, 400) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 408), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 416)); + WC_S64(r15, 408) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 416), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 424)); + WC_S64(r15, 416) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 424), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 432)); + WC_S64(r15, 424) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 432), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 440)); + WC_S64(r15, 432) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 440), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 448)); + WC_S64(r15, 440) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 448), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 456)); + WC_S64(r15, 448) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 456), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 464)); + WC_S64(r15, 456) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 464), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 472)); + WC_S64(r15, 464) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 472), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 480)); + WC_S64(r15, 472) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 480), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 488)); + WC_S64(r15, 480) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 488), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 496)); + WC_S64(r15, 488) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 496), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 504)); + WC_S64(r15, 496) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 504), (unsigned long long*)&rax); + WC_S64(r15, 504) = (word64)(rax); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 768) = (word64)(r9); + r15 = (word64)(r15 + 0x100); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 200)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 208)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 216)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 224)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 232)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 240)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 248)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 256)); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 256), (unsigned long long*)&r8); + WC_S64(r15, 256) = (word64)(r8); + /* Add to zero */ + rax = (word64)(WC_L64(r11, 264)); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 272)); + WC_S64(r15, 264) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 280)); + WC_S64(r15, 272) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 288)); + WC_S64(r15, 280) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 296)); + WC_S64(r15, 288) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 304)); + WC_S64(r15, 296) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 312)); + WC_S64(r15, 304) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 320)); + WC_S64(r15, 312) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 328)); + WC_S64(r15, 320) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 336)); + WC_S64(r15, 328) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 344)); + WC_S64(r15, 336) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 352)); + WC_S64(r15, 344) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 360)); + WC_S64(r15, 352) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 368)); + WC_S64(r15, 360) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 376)); + WC_S64(r15, 368) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 384)); + WC_S64(r15, 376) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 392)); + WC_S64(r15, 384) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 400)); + WC_S64(r15, 392) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 408)); + WC_S64(r15, 400) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 416)); + WC_S64(r15, 408) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 424)); + WC_S64(r15, 416) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 432)); + WC_S64(r15, 424) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 440)); + WC_S64(r15, 432) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 448)); + WC_S64(r15, 440) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 456)); + WC_S64(r15, 448) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 464)); + WC_S64(r15, 456) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 472)); + WC_S64(r15, 464) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 480)); + WC_S64(r15, 472) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 488)); + WC_S64(r15, 480) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 496)); + WC_S64(r15, 488) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 504)); + WC_S64(r15, 496) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + WC_S64(r15, 504) = (word64)(rax); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_4096_mul_avx2_64(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rsp, r10, r12, rax, r13, rcx, r8, r11, r14, r15, r9; + XALIGNED(32) WC_X64I_SLOT stk[200]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 1600; + rsp = (word64)(rsp - 1576); + WC_S64(rsp, 1536) = (word64)(rdi); + WC_S64(rsp, 1544) = (word64)(rsi); + WC_S64(rsp, 1552) = (word64)(rdx); + r10 = (word64)(rsp + 1024); + r12 = (word64)(rsi + 256); + /* Add */ + rax = (word64)(WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rsi, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rsi, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 248), (unsigned long long*)&rcx); + WC_S64(r10, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 1560) = (word64)(r13); + r11 = (word64)(rsp + 1280); + r12 = (word64)(rdx + 256); + /* Add */ + rax = (word64)(WC_L64(rdx, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, rax, WC_L64(r12, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 8)); + WC_S64(r11, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 16)); + WC_S64(r11, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 24)); + WC_S64(r11, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 32)); + WC_S64(r11, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 40)); + WC_S64(r11, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 48)); + WC_S64(r11, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 56)); + WC_S64(r11, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 64)); + WC_S64(r11, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 72)); + WC_S64(r11, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 80)); + WC_S64(r11, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 88)); + WC_S64(r11, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 96)); + WC_S64(r11, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 104)); + WC_S64(r11, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 112)); + WC_S64(r11, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 120)); + WC_S64(r11, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 128)); + WC_S64(r11, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 136)); + WC_S64(r11, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 144)); + WC_S64(r11, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 152)); + WC_S64(r11, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 160)); + WC_S64(r11, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 168)); + WC_S64(r11, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 176)); + WC_S64(r11, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 184)); + WC_S64(r11, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 192)); + WC_S64(r11, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 200)); + WC_S64(r11, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 208)); + WC_S64(r11, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 216)); + WC_S64(r11, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 224)); + WC_S64(r11, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rdx, 232)); + WC_S64(r11, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r12, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(rdx, 240)); + WC_S64(r11, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r12, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(rdx, 248)); + WC_S64(r11, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r12, 248), (unsigned long long*)&rcx); + WC_S64(r11, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rsp, 1568) = (word64)(r14); + rdx = (word64)(r11); + rsi = (word64)(r10); + rdi = (word64)(rsp); + (void)sp_2048_mul_avx2_32((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 1552)); + rsi = (word64)(WC_L64(rsp, 1544)); + rdi = (word64)(rsp + 512); + rdx = (word64)(rdx + 0x100); + rsi = (word64)(rsi + 0x100); + (void)sp_2048_mul_avx2_32((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); + rdx = (word64)(WC_L64(rsp, 1552)); + rsi = (word64)(WC_L64(rsp, 1544)); + rdi = (word64)(WC_L64(rsp, 1536)); + (void)sp_2048_mul_avx2_32((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx); +#ifdef _WIN64 + rdx = (word64)(WC_L64(rsp, 1552)); + rsi = (word64)(WC_L64(rsp, 1544)); + rdi = (word64)(WC_L64(rsp, 1536)); +#endif /* _WIN64 */ + r13 = (word64)(WC_L64(rsp, 1560)); + r14 = (word64)(WC_L64(rsp, 1568)); + r15 = (word64)(WC_L64(rsp, 1536)); + r9 = (word64)(r13); + r10 = (word64)(rsp + 1024); + r11 = (word64)(rsp + 1280); + r9 = (word64)(r9 & r14); + r13 = (word64)(0 - r13); + r14 = (word64)(0 - r14); + r15 = (word64)(r15 + 0x200); + rax = (word64)(WC_L64(r10, 0)); + rcx = (word64)(WC_L64(r11, 0)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + r8 = (word64)(WC_L64(r11, 8)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + rax = (word64)(WC_L64(r11, 16)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + rcx = (word64)(WC_L64(r11, 24)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + r8 = (word64)(WC_L64(r11, 32)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + rax = (word64)(WC_L64(r11, 40)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + rcx = (word64)(WC_L64(r11, 48)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + r8 = (word64)(WC_L64(r11, 56)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + rax = (word64)(WC_L64(r11, 64)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + rcx = (word64)(WC_L64(r11, 72)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + r8 = (word64)(WC_L64(r11, 80)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + rax = (word64)(WC_L64(r11, 88)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + rcx = (word64)(WC_L64(r11, 96)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + r8 = (word64)(WC_L64(r11, 104)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + rax = (word64)(WC_L64(r11, 112)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + rcx = (word64)(WC_L64(r11, 120)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + r8 = (word64)(WC_L64(r11, 128)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + rax = (word64)(WC_L64(r11, 136)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + rcx = (word64)(WC_L64(r11, 144)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + r8 = (word64)(WC_L64(r11, 152)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + rax = (word64)(WC_L64(r11, 160)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + rcx = (word64)(WC_L64(r11, 168)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + r8 = (word64)(WC_L64(r11, 176)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + rax = (word64)(WC_L64(r11, 184)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + rcx = (word64)(WC_L64(r11, 192)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + r8 = (word64)(WC_L64(r11, 200)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + rax = (word64)(WC_L64(r11, 208)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + rcx = (word64)(WC_L64(r11, 216)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + r8 = (word64)(WC_L64(r11, 224)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + rax = (word64)(WC_L64(r11, 232)); + r8 = (word64)((word64)_pext_u64(r8, r14)); + rax = (word64)((word64)_pext_u64(rax, r13)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + rcx = (word64)(WC_L64(r11, 240)); + rax = (word64)((word64)_pext_u64(rax, r14)); + rcx = (word64)((word64)_pext_u64(rcx, r13)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, rcx, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + r8 = (word64)(WC_L64(r11, 248)); + rcx = (word64)((word64)_pext_u64(rcx, r14)); + r8 = (word64)((word64)_pext_u64(r8, r13)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, r8, (unsigned long long*)&rcx); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + r11 = (word64)(rsp + 512); + r10 = (word64)(rsp); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 256)); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 264)); + WC_S64(r10, 256) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 272)); + WC_S64(r10, 264) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 280)); + WC_S64(r10, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 288)); + WC_S64(r10, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 296)); + WC_S64(r10, 288) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 304)); + WC_S64(r10, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 312)); + WC_S64(r10, 304) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 320)); + WC_S64(r10, 312) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 328)); + WC_S64(r10, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 336)); + WC_S64(r10, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 344)); + WC_S64(r10, 336) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 352)); + WC_S64(r10, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 360)); + WC_S64(r10, 352) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 368)); + WC_S64(r10, 360) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 376)); + WC_S64(r10, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 376), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 384)); + WC_S64(r10, 376) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 384), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 392)); + WC_S64(r10, 384) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 392), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 400)); + WC_S64(r10, 392) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 400), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 408)); + WC_S64(r10, 400) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 408), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 416)); + WC_S64(r10, 408) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 416), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 424)); + WC_S64(r10, 416) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 424), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 432)); + WC_S64(r10, 424) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 432), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 440)); + WC_S64(r10, 432) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 440), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 448)); + WC_S64(r10, 440) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 448), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 456)); + WC_S64(r10, 448) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 456), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 464)); + WC_S64(r10, 456) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 464), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 472)); + WC_S64(r10, 464) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 472), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 480)); + WC_S64(r10, 472) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 480), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 488)); + WC_S64(r10, 480) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(r11, 488), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 496)); + WC_S64(r10, 488) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(r11, 496), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 504)); + WC_S64(r10, 496) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(r11, 504), (unsigned long long*)&rax); + WC_S64(r10, 504) = (word64)(rax); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + rax = (word64)(WC_L64(r10, 0)); + cf = _subborrow_u64(0, rax, WC_L64(rdi, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 8)); + WC_S64(r10, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 16)); + WC_S64(r10, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 24)); + WC_S64(r10, 16) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 32)); + WC_S64(r10, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 40)); + WC_S64(r10, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 48)); + WC_S64(r10, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 56)); + WC_S64(r10, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 64)); + WC_S64(r10, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 72)); + WC_S64(r10, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 80)); + WC_S64(r10, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 88)); + WC_S64(r10, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 96)); + WC_S64(r10, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 104)); + WC_S64(r10, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 112)); + WC_S64(r10, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 120)); + WC_S64(r10, 112) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 128)); + WC_S64(r10, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 136)); + WC_S64(r10, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 144)); + WC_S64(r10, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 152)); + WC_S64(r10, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 160)); + WC_S64(r10, 152) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 168)); + WC_S64(r10, 160) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 176)); + WC_S64(r10, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 184)); + WC_S64(r10, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 192)); + WC_S64(r10, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 200)); + WC_S64(r10, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 208)); + WC_S64(r10, 200) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 216)); + WC_S64(r10, 208) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 224)); + WC_S64(r10, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 232)); + WC_S64(r10, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 240)); + WC_S64(r10, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 248)); + WC_S64(r10, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 256)); + WC_S64(r10, 248) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 264)); + WC_S64(r10, 256) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 272)); + WC_S64(r10, 264) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 280)); + WC_S64(r10, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 288)); + WC_S64(r10, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 296)); + WC_S64(r10, 288) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 304)); + WC_S64(r10, 296) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 312)); + WC_S64(r10, 304) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 320)); + WC_S64(r10, 312) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 328)); + WC_S64(r10, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 336)); + WC_S64(r10, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 344)); + WC_S64(r10, 336) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 352)); + WC_S64(r10, 344) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 360)); + WC_S64(r10, 352) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 368)); + WC_S64(r10, 360) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 376)); + WC_S64(r10, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 376), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 384)); + WC_S64(r10, 376) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 384), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 392)); + WC_S64(r10, 384) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 392), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 400)); + WC_S64(r10, 392) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 400), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 408)); + WC_S64(r10, 400) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 408), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 416)); + WC_S64(r10, 408) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 416), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 424)); + WC_S64(r10, 416) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 424), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 432)); + WC_S64(r10, 424) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 432), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 440)); + WC_S64(r10, 432) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 440), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 448)); + WC_S64(r10, 440) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 448), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 456)); + WC_S64(r10, 448) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 456), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 464)); + WC_S64(r10, 456) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 464), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 472)); + WC_S64(r10, 464) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 472), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 480)); + WC_S64(r10, 472) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 480), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r10, 488)); + WC_S64(r10, 480) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdi, 488), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r10, 496)); + WC_S64(r10, 488) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdi, 496), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r10, 504)); + WC_S64(r10, 496) = (word64)(r8); + cf = _subborrow_u64(cf, rax, WC_L64(rdi, 504), (unsigned long long*)&rax); + WC_S64(r10, 504) = (word64)(rax); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + r15 = (word64)(r15 - 0x100); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r10, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 200)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 208)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 216)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 224)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 232)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 240)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 248)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 256)); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 256), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 264)); + WC_S64(r15, 256) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 264), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 272)); + WC_S64(r15, 264) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 280)); + WC_S64(r15, 272) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 280), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 288)); + WC_S64(r15, 280) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 288), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 296)); + WC_S64(r15, 288) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 296), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 304)); + WC_S64(r15, 296) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 304), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 312)); + WC_S64(r15, 304) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 312), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 320)); + WC_S64(r15, 312) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 328)); + WC_S64(r15, 320) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 328), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 336)); + WC_S64(r15, 328) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 336), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 344)); + WC_S64(r15, 336) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 344), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 352)); + WC_S64(r15, 344) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 352), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 360)); + WC_S64(r15, 352) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 360), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 368)); + WC_S64(r15, 360) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 376)); + WC_S64(r15, 368) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 376), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 384)); + WC_S64(r15, 376) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 384), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 392)); + WC_S64(r15, 384) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 392), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 400)); + WC_S64(r15, 392) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 400), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 408)); + WC_S64(r15, 400) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 408), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 416)); + WC_S64(r15, 408) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 416), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 424)); + WC_S64(r15, 416) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 424), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 432)); + WC_S64(r15, 424) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 432), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 440)); + WC_S64(r15, 432) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 440), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 448)); + WC_S64(r15, 440) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 448), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 456)); + WC_S64(r15, 448) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 456), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 464)); + WC_S64(r15, 456) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 464), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 472)); + WC_S64(r15, 464) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 472), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 480)); + WC_S64(r15, 472) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 480), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 488)); + WC_S64(r15, 480) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r10, 488), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 496)); + WC_S64(r15, 488) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r10, 496), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 504)); + WC_S64(r15, 496) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r10, 504), (unsigned long long*)&rax); + WC_S64(r15, 504) = (word64)(rax); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 768) = (word64)(r9); + r15 = (word64)(r15 + 0x100); + /* Add */ + rax = (word64)(WC_L64(r15, 0)); + cf = _addcarry_u64(0, rax, WC_L64(r11, 0), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 8)); + WC_S64(r15, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 8), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 16)); + WC_S64(r15, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 16), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 24)); + WC_S64(r15, 16) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 24), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 32)); + WC_S64(r15, 24) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 40)); + WC_S64(r15, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 40), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 48)); + WC_S64(r15, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 48), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 56)); + WC_S64(r15, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 56), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 64)); + WC_S64(r15, 56) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 64), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 72)); + WC_S64(r15, 64) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 72), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 80)); + WC_S64(r15, 72) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 88)); + WC_S64(r15, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 88), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 96)); + WC_S64(r15, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 96), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 104)); + WC_S64(r15, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 104), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 112)); + WC_S64(r15, 104) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 112), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 120)); + WC_S64(r15, 112) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 120), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 128)); + WC_S64(r15, 120) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 136)); + WC_S64(r15, 128) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 136), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 144)); + WC_S64(r15, 136) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 144), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 152)); + WC_S64(r15, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 152), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 160)); + WC_S64(r15, 152) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 160), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 168)); + WC_S64(r15, 160) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 168), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 176)); + WC_S64(r15, 168) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 184)); + WC_S64(r15, 176) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 184), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 192)); + WC_S64(r15, 184) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 192), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 200)); + WC_S64(r15, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 200), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 208)); + WC_S64(r15, 200) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 208), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 216)); + WC_S64(r15, 208) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 216), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 224)); + WC_S64(r15, 216) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 232)); + WC_S64(r15, 224) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 232), (unsigned long long*)&r8); + rax = (word64)(WC_L64(r15, 240)); + WC_S64(r15, 232) = (word64)(r8); + cf = _addcarry_u64(cf, rax, WC_L64(r11, 240), (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r15, 248)); + WC_S64(r15, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, WC_L64(r11, 248), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r15, 256)); + WC_S64(r15, 248) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(r11, 256), (unsigned long long*)&r8); + WC_S64(r15, 256) = (word64)(r8); + /* Add to zero */ + rax = (word64)(WC_L64(r11, 264)); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 272)); + WC_S64(r15, 264) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 280)); + WC_S64(r15, 272) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 288)); + WC_S64(r15, 280) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 296)); + WC_S64(r15, 288) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 304)); + WC_S64(r15, 296) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 312)); + WC_S64(r15, 304) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 320)); + WC_S64(r15, 312) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 328)); + WC_S64(r15, 320) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 336)); + WC_S64(r15, 328) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 344)); + WC_S64(r15, 336) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 352)); + WC_S64(r15, 344) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 360)); + WC_S64(r15, 352) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 368)); + WC_S64(r15, 360) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 376)); + WC_S64(r15, 368) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 384)); + WC_S64(r15, 376) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 392)); + WC_S64(r15, 384) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 400)); + WC_S64(r15, 392) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 408)); + WC_S64(r15, 400) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 416)); + WC_S64(r15, 408) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 424)); + WC_S64(r15, 416) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 432)); + WC_S64(r15, 424) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 440)); + WC_S64(r15, 432) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 448)); + WC_S64(r15, 440) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 456)); + WC_S64(r15, 448) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 464)); + WC_S64(r15, 456) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 472)); + WC_S64(r15, 464) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 480)); + WC_S64(r15, 472) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rcx = (word64)(WC_L64(r11, 488)); + WC_S64(r15, 480) = (word64)(rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(r11, 496)); + WC_S64(r15, 488) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + rax = (word64)(WC_L64(r11, 504)); + WC_S64(r15, 496) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + WC_S64(r15, 504) = (word64)(rax); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * Karatsuba: ah^2, al^2, (al - ah)^2 + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_4096_sqr_64(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rcx, r8, r9, rdx, rax; + XALIGNED(32) WC_X64I_SLOT stk[68]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 544; + rsp = (word64)(rsp - 528); + WC_S64(rsp, 512) = (word64)(rdi); + WC_S64(rsp, 520) = (word64)(rsi); + rcx = (word64)(0); + r8 = (word64)(rsp); + r9 = (word64)(rsi + 256); + rdx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(r9, 0), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 8)); + WC_S64(r8, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 8), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 16)); + WC_S64(r8, 8) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 16), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r8, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 24), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 32)); + WC_S64(r8, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 32), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 40)); + WC_S64(r8, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 40), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 48)); + WC_S64(r8, 40) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 48), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 56)); + WC_S64(r8, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 56), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 64)); + WC_S64(r8, 56) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 64), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r8, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 72), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 80)); + WC_S64(r8, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 80), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 88)); + WC_S64(r8, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 88), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 96)); + WC_S64(r8, 88) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 96), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 104)); + WC_S64(r8, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 104), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 112)); + WC_S64(r8, 104) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 112), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r8, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 120), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 128)); + WC_S64(r8, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 128), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 136)); + WC_S64(r8, 128) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 136), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 144)); + WC_S64(r8, 136) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 144), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 152)); + WC_S64(r8, 144) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 152), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 160)); + WC_S64(r8, 152) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 160), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 168)); + WC_S64(r8, 160) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 168), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 176)); + WC_S64(r8, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 176), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 184)); + WC_S64(r8, 176) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 184), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 192)); + WC_S64(r8, 184) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 192), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 200)); + WC_S64(r8, 192) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 200), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 208)); + WC_S64(r8, 200) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 208), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 216)); + WC_S64(r8, 208) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 216), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 224)); + WC_S64(r8, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 224), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 232)); + WC_S64(r8, 224) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 232), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 240)); + WC_S64(r8, 232) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 240), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 248)); + WC_S64(r8, 240) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 248), (unsigned long long*)&rax); + WC_S64(r8, 248) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* Cond Negate */ + rdx = (word64)(WC_L64(r8, 0)); + r9 = (word64)(rcx); + rdx = (word64)(rdx ^ rcx); + r9 = (word64)(0 - r9); + cf = _subborrow_u64(0, rdx, rcx, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 8)); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(r8, 0) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 16)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 8) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 24)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 16) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 32)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 24) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 40)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 32) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 48)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 40) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 56)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 48) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 64)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 56) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 72)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 64) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 80)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 72) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 88)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 80) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 96)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 88) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 104)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 96) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 112)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 104) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 120)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 112) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 128)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 120) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 136)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 128) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 144)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 136) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 152)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 144) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 160)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 152) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 168)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 160) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 176)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 168) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 184)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 176) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 192)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 184) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 200)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 192) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 208)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 200) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 216)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 208) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 224)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 216) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 232)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 224) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 240)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 232) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 248)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 240) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + rax = (word64)(rax + r9); + WC_S64(r8, 248) = (word64)(rax); + rsi = (word64)(r8); + rdi = (word64)(rsp); + (void)sp_2048_sqr_32((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); + rsi = (word64)(WC_L64(rsp, 520)); + rdi = (word64)(WC_L64(rsp, 512)); + rsi = (word64)(rsi + 0x100); + rdi = (word64)(rdi + 0x200); + (void)sp_2048_sqr_32((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); + rsi = (word64)(WC_L64(rsp, 520)); + rdi = (word64)(WC_L64(rsp, 512)); + (void)sp_2048_sqr_32((sp_digit*)(size_t)rdi, (const sp_digit*)(size_t)rsi); +#ifdef _WIN64 + rsi = (word64)(WC_L64(rsp, 520)); + rdi = (word64)(WC_L64(rsp, 512)); +#endif /* _WIN64 */ + rsi = (word64)(WC_L64(rsp, 512)); + r8 = (word64)(rsp + 256); + rsi = (word64)(rsi + 0x300); + rcx = (word64)(0); + rax = (word64)(WC_L64(r8, -256)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -256), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -248)); + WC_S64(r8, -256) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -248), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -240)); + WC_S64(r8, -248) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -232)); + WC_S64(r8, -240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -224)); + WC_S64(r8, -232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -216)); + WC_S64(r8, -224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -208)); + WC_S64(r8, -216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -200)); + WC_S64(r8, -208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -192)); + WC_S64(r8, -200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -184)); + WC_S64(r8, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -176)); + WC_S64(r8, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -168)); + WC_S64(r8, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -160)); + WC_S64(r8, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -152)); + WC_S64(r8, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -144)); + WC_S64(r8, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -136)); + WC_S64(r8, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -128)); + WC_S64(r8, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 128)); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 136)); + WC_S64(r8, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 144)); + WC_S64(r8, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 152)); + WC_S64(r8, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 160)); + WC_S64(r8, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 168)); + WC_S64(r8, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 176)); + WC_S64(r8, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 184)); + WC_S64(r8, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 192)); + WC_S64(r8, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 200)); + WC_S64(r8, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 208)); + WC_S64(r8, 200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 216)); + WC_S64(r8, 208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 224)); + WC_S64(r8, 216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 232)); + WC_S64(r8, 224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 240)); + WC_S64(r8, 232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 248)); + WC_S64(r8, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 248), (unsigned long long*)&rdx); + WC_S64(r8, 248) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rsi = (word64)(rsi - 0x200); + rax = (word64)(WC_L64(r8, -256)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -256), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -248)); + WC_S64(r8, -256) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -248), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -240)); + WC_S64(r8, -248) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -232)); + WC_S64(r8, -240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -224)); + WC_S64(r8, -232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -216)); + WC_S64(r8, -224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -208)); + WC_S64(r8, -216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -200)); + WC_S64(r8, -208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -192)); + WC_S64(r8, -200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -184)); + WC_S64(r8, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -176)); + WC_S64(r8, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -168)); + WC_S64(r8, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -160)); + WC_S64(r8, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -152)); + WC_S64(r8, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -144)); + WC_S64(r8, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -136)); + WC_S64(r8, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -128)); + WC_S64(r8, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 128)); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 136)); + WC_S64(r8, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 144)); + WC_S64(r8, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 152)); + WC_S64(r8, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 160)); + WC_S64(r8, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 168)); + WC_S64(r8, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 176)); + WC_S64(r8, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 184)); + WC_S64(r8, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 192)); + WC_S64(r8, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 200)); + WC_S64(r8, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 208)); + WC_S64(r8, 200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 216)); + WC_S64(r8, 208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 224)); + WC_S64(r8, 216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 232)); + WC_S64(r8, 224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 240)); + WC_S64(r8, 232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 248)); + WC_S64(r8, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 248), (unsigned long long*)&rdx); + WC_S64(r8, 248) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 512)); + rcx = (word64)(0 - rcx); + rdi = (word64)(rdi + 0x200); + rax = (word64)(WC_L64(rdi, -256)); + cf = _subborrow_u64(0, rax, WC_L64(r8, -256), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -248)); + WC_S64(rdi, -256) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -248), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -240)); + WC_S64(rdi, -248) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -232)); + WC_S64(rdi, -240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -224)); + WC_S64(rdi, -232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -216)); + WC_S64(rdi, -224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -208)); + WC_S64(rdi, -216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -200)); + WC_S64(rdi, -208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -192)); + WC_S64(rdi, -200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -184)); + WC_S64(rdi, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -176)); + WC_S64(rdi, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -168)); + WC_S64(rdi, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -160)); + WC_S64(rdi, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -152)); + WC_S64(rdi, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -144)); + WC_S64(rdi, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -136)); + WC_S64(rdi, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -128)); + WC_S64(rdi, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -120)); + WC_S64(rdi, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -112)); + WC_S64(rdi, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -104)); + WC_S64(rdi, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -96)); + WC_S64(rdi, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -88)); + WC_S64(rdi, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -80)); + WC_S64(rdi, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -72)); + WC_S64(rdi, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -64)); + WC_S64(rdi, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -56)); + WC_S64(rdi, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -48)); + WC_S64(rdi, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -40)); + WC_S64(rdi, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -32)); + WC_S64(rdi, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -24)); + WC_S64(rdi, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -16)); + WC_S64(rdi, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -8)); + WC_S64(rdi, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 0)); + WC_S64(rdi, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 192)); + WC_S64(rdi, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 200)); + WC_S64(rdi, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 208)); + WC_S64(rdi, 200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 216)); + WC_S64(rdi, 208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 224)); + WC_S64(rdi, 216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 232)); + WC_S64(rdi, 224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 240)); + WC_S64(rdi, 232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 248)); + WC_S64(rdi, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 248), (unsigned long long*)&rdx); + WC_S64(rdi, 248) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 512)); + rdi = (word64)(rdi + 0x300); + /* Add in word */ + rax = (word64)(WC_L64(rdi, 0)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 192)); + WC_S64(rdi, 184) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 200)); + WC_S64(rdi, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 208)); + WC_S64(rdi, 200) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 216)); + WC_S64(rdi, 208) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 224)); + WC_S64(rdi, 216) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 232)); + WC_S64(rdi, 224) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 240)); + WC_S64(rdi, 232) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 248)); + WC_S64(rdi, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 248) = (word64)(rdx); + rsi = (word64)(WC_L64(rsp, 520)); + rdi = (word64)(WC_L64(rsp, 512)); +} + +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * Karatsuba: ah^2, al^2, (al - ah)^2 + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_4096_sqr_avx2_64(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rcx, r8, r9, rdx, rax; + XALIGNED(32) WC_X64I_SLOT stk[68]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 544; + rsp = (word64)(rsp - 528); + WC_S64(rsp, 512) = (word64)(rdi); + WC_S64(rsp, 520) = (word64)(rsi); + rcx = (word64)(0); + r8 = (word64)(rsp); + r9 = (word64)(rsi + 256); + rdx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(r9, 0), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 8)); + WC_S64(r8, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 8), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 16)); + WC_S64(r8, 8) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 16), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 24)); + WC_S64(r8, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 24), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 32)); + WC_S64(r8, 24) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 32), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 40)); + WC_S64(r8, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 40), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 48)); + WC_S64(r8, 40) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 48), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 56)); + WC_S64(r8, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 56), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 64)); + WC_S64(r8, 56) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 64), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 72)); + WC_S64(r8, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 72), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 80)); + WC_S64(r8, 72) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 80), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 88)); + WC_S64(r8, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 88), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 96)); + WC_S64(r8, 88) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 96), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 104)); + WC_S64(r8, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 104), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 112)); + WC_S64(r8, 104) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 112), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 120)); + WC_S64(r8, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 120), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 128)); + WC_S64(r8, 120) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 128), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 136)); + WC_S64(r8, 128) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 136), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 144)); + WC_S64(r8, 136) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 144), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 152)); + WC_S64(r8, 144) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 152), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 160)); + WC_S64(r8, 152) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 160), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 168)); + WC_S64(r8, 160) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 168), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 176)); + WC_S64(r8, 168) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 176), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 184)); + WC_S64(r8, 176) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 184), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 192)); + WC_S64(r8, 184) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 192), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 200)); + WC_S64(r8, 192) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 200), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 208)); + WC_S64(r8, 200) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 208), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 216)); + WC_S64(r8, 208) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 216), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 224)); + WC_S64(r8, 216) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 224), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 232)); + WC_S64(r8, 224) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 232), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rsi, 240)); + WC_S64(r8, 232) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r9, 240), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rsi, 248)); + WC_S64(r8, 240) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r9, 248), (unsigned long long*)&rax); + WC_S64(r8, 248) = (word64)(rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* Cond Negate */ + rdx = (word64)(WC_L64(r8, 0)); + r9 = (word64)(rcx); + rdx = (word64)(rdx ^ rcx); + r9 = (word64)(0 - r9); + cf = _subborrow_u64(0, rdx, rcx, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 8)); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(r8, 0) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 16)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 8) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 24)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 16) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 32)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 24) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 40)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 32) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 48)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 40) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 56)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 48) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 64)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 56) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 72)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 64) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 80)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 72) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 88)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 80) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 96)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 88) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 104)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 96) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 112)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 104) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 120)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 112) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 128)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 120) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 136)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 128) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 144)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 136) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 152)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 144) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 160)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 152) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 168)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 160) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 176)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 168) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 184)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 176) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 192)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 184) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 200)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 192) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 208)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 200) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 216)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 208) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 224)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 216) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 232)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 224) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 240)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 232) = (word64)(rax); + rdx = (word64)(rdx ^ rcx); + cf = _addcarry_u64(0, rdx, r9, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 248)); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)((byte)(cf)) & 0xff); + WC_S64(r8, 240) = (word64)(rdx); + rax = (word64)(rax ^ rcx); + rax = (word64)(rax + r9); + WC_S64(r8, 248) = (word64)(rax); + rsi = (word64)(r8); + rdi = (word64)(rsp); + (void)sp_2048_sqr_avx2_32((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); + rsi = (word64)(WC_L64(rsp, 520)); + rdi = (word64)(WC_L64(rsp, 512)); + rsi = (word64)(rsi + 0x100); + rdi = (word64)(rdi + 0x200); + (void)sp_2048_sqr_avx2_32((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); + rsi = (word64)(WC_L64(rsp, 520)); + rdi = (word64)(WC_L64(rsp, 512)); + (void)sp_2048_sqr_avx2_32((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi); +#ifdef _WIN64 + rsi = (word64)(WC_L64(rsp, 520)); + rdi = (word64)(WC_L64(rsp, 512)); +#endif /* _WIN64 */ + rsi = (word64)(WC_L64(rsp, 512)); + r8 = (word64)(rsp + 256); + rsi = (word64)(rsi + 0x300); + rcx = (word64)(0); + rax = (word64)(WC_L64(r8, -256)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -256), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -248)); + WC_S64(r8, -256) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -248), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -240)); + WC_S64(r8, -248) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -232)); + WC_S64(r8, -240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -224)); + WC_S64(r8, -232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -216)); + WC_S64(r8, -224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -208)); + WC_S64(r8, -216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -200)); + WC_S64(r8, -208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -192)); + WC_S64(r8, -200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -184)); + WC_S64(r8, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -176)); + WC_S64(r8, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -168)); + WC_S64(r8, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -160)); + WC_S64(r8, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -152)); + WC_S64(r8, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -144)); + WC_S64(r8, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -136)); + WC_S64(r8, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -128)); + WC_S64(r8, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 128)); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 136)); + WC_S64(r8, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 144)); + WC_S64(r8, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 152)); + WC_S64(r8, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 160)); + WC_S64(r8, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 168)); + WC_S64(r8, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 176)); + WC_S64(r8, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 184)); + WC_S64(r8, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 192)); + WC_S64(r8, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 200)); + WC_S64(r8, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 208)); + WC_S64(r8, 200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 216)); + WC_S64(r8, 208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 224)); + WC_S64(r8, 216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 232)); + WC_S64(r8, 224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 240)); + WC_S64(r8, 232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 248)); + WC_S64(r8, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 248), (unsigned long long*)&rdx); + WC_S64(r8, 248) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rsi = (word64)(rsi - 0x200); + rax = (word64)(WC_L64(r8, -256)); + cf = _subborrow_u64(0, rax, WC_L64(rsi, -256), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -248)); + WC_S64(r8, -256) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -248), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -240)); + WC_S64(r8, -248) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -232)); + WC_S64(r8, -240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -224)); + WC_S64(r8, -232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -216)); + WC_S64(r8, -224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -208)); + WC_S64(r8, -216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -200)); + WC_S64(r8, -208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -192)); + WC_S64(r8, -200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -184)); + WC_S64(r8, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -176)); + WC_S64(r8, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -168)); + WC_S64(r8, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -160)); + WC_S64(r8, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -152)); + WC_S64(r8, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -144)); + WC_S64(r8, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -136)); + WC_S64(r8, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -128)); + WC_S64(r8, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -120)); + WC_S64(r8, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -112)); + WC_S64(r8, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -104)); + WC_S64(r8, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -96)); + WC_S64(r8, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -88)); + WC_S64(r8, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -80)); + WC_S64(r8, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -72)); + WC_S64(r8, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -64)); + WC_S64(r8, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -56)); + WC_S64(r8, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -48)); + WC_S64(r8, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -40)); + WC_S64(r8, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -32)); + WC_S64(r8, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -24)); + WC_S64(r8, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, -16)); + WC_S64(r8, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, -8)); + WC_S64(r8, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 0)); + WC_S64(r8, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 8)); + WC_S64(r8, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 16)); + WC_S64(r8, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 24)); + WC_S64(r8, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 32)); + WC_S64(r8, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 40)); + WC_S64(r8, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 48)); + WC_S64(r8, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 56)); + WC_S64(r8, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 64)); + WC_S64(r8, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 72)); + WC_S64(r8, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 80)); + WC_S64(r8, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 88)); + WC_S64(r8, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 96)); + WC_S64(r8, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 104)); + WC_S64(r8, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 112)); + WC_S64(r8, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 120)); + WC_S64(r8, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 128)); + WC_S64(r8, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 136)); + WC_S64(r8, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 144)); + WC_S64(r8, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 152)); + WC_S64(r8, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 160)); + WC_S64(r8, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 168)); + WC_S64(r8, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 176)); + WC_S64(r8, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 184)); + WC_S64(r8, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 192)); + WC_S64(r8, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 200)); + WC_S64(r8, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 208)); + WC_S64(r8, 200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 216)); + WC_S64(r8, 208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 224)); + WC_S64(r8, 216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 232)); + WC_S64(r8, 224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(r8, 240)); + WC_S64(r8, 232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(rsi, 240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(r8, 248)); + WC_S64(r8, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 248), (unsigned long long*)&rdx); + WC_S64(r8, 248) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 512)); + rcx = (word64)(0 - rcx); + rdi = (word64)(rdi + 0x200); + rax = (word64)(WC_L64(rdi, -256)); + cf = _subborrow_u64(0, rax, WC_L64(r8, -256), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -248)); + WC_S64(rdi, -256) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -248), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -240)); + WC_S64(rdi, -248) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -232)); + WC_S64(rdi, -240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -224)); + WC_S64(rdi, -232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -216)); + WC_S64(rdi, -224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -208)); + WC_S64(rdi, -216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -200)); + WC_S64(rdi, -208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -192)); + WC_S64(rdi, -200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -184)); + WC_S64(rdi, -192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -176)); + WC_S64(rdi, -184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -168)); + WC_S64(rdi, -176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -160)); + WC_S64(rdi, -168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -152)); + WC_S64(rdi, -160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -144)); + WC_S64(rdi, -152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -136)); + WC_S64(rdi, -144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -128)); + WC_S64(rdi, -136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -120)); + WC_S64(rdi, -128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -112)); + WC_S64(rdi, -120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -104)); + WC_S64(rdi, -112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -96)); + WC_S64(rdi, -104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -88)); + WC_S64(rdi, -96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -80)); + WC_S64(rdi, -88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -72)); + WC_S64(rdi, -80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -64)); + WC_S64(rdi, -72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -56)); + WC_S64(rdi, -64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -48)); + WC_S64(rdi, -56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -40)); + WC_S64(rdi, -48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -32)); + WC_S64(rdi, -40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -24)); + WC_S64(rdi, -32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, -16)); + WC_S64(rdi, -24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, -16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, -8)); + WC_S64(rdi, -16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, -8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 0)); + WC_S64(rdi, -8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 0), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 8), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 16), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 24), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 32), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 40), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 48), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 56), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 64), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 72), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 80), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 88), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 96), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 104), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 112), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 120), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 128), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 136), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 144), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 152), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 160), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 168), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 176), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 184), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 192)); + WC_S64(rdi, 184) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 192), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 200)); + WC_S64(rdi, 192) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 200), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 208)); + WC_S64(rdi, 200) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 208), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 216)); + WC_S64(rdi, 208) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 216), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 224)); + WC_S64(rdi, 216) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 224), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 232)); + WC_S64(rdi, 224) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 232), (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 240)); + WC_S64(rdi, 232) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, WC_L64(r8, 240), (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 248)); + WC_S64(rdi, 240) = (word64)(rax); + cf = _subborrow_u64(cf, rdx, WC_L64(r8, 248), (unsigned long long*)&rdx); + WC_S64(rdi, 248) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + rdi = (word64)(WC_L64(rsp, 512)); + rdi = (word64)(rdi + 0x300); + /* Add in word */ + rax = (word64)(WC_L64(rdi, 0)); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 128)); + WC_S64(rdi, 120) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 136)); + WC_S64(rdi, 128) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 144)); + WC_S64(rdi, 136) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 152)); + WC_S64(rdi, 144) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 160)); + WC_S64(rdi, 152) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 168)); + WC_S64(rdi, 160) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 176)); + WC_S64(rdi, 168) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 184)); + WC_S64(rdi, 176) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 192)); + WC_S64(rdi, 184) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 200)); + WC_S64(rdi, 192) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 208)); + WC_S64(rdi, 200) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 216)); + WC_S64(rdi, 208) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 224)); + WC_S64(rdi, 216) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 232)); + WC_S64(rdi, 224) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rax = (word64)(WC_L64(rdi, 240)); + WC_S64(rdi, 232) = (word64)(rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + rdx = (word64)(WC_L64(rdi, 248)); + WC_S64(rdi, 240) = (word64)(rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + WC_S64(rdi, 248) = (word64)(rdx); + rsi = (word64)(WC_L64(rsp, 520)); + rdi = (word64)(WC_L64(rsp, 512)); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_4096_mul_d_64(sp_digit* r, const sp_digit* a, sp_digit b) +{ + word64 rdi, rsi, rcx, rax, r10, rdx = 0, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)b; + + /* A[0] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + WC_S64(rdi, 0) = (word64)(r8); + /* A[1] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 88) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 104) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 112) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[16] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 128)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[17] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 136)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 136) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[18] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 144)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 144) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[19] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 152)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 152) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[20] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 160)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 160) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[21] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 168)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[22] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 176)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[23] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 184)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 184) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[24] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 192)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 192) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[25] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 200)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 200) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[26] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 208)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 208) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[27] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 216)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 216) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[28] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 224)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 224) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[29] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 232)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 232) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[30] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 240)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 240) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[31] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 248)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 248) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[32] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 256)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 256) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[33] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 264)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 264) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[34] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 272)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 272) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[35] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 280)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 280) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[36] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 288)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 288) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[37] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 296)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 296) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[38] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 304)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 304) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[39] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 312)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 312) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[40] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 320)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 320) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[41] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 328)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 328) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[42] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 336)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 336) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[43] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 344)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 344) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[44] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 352)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 352) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[45] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 360)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 360) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[46] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 368)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 368) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[47] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 376)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 376) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[48] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 384)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 384) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[49] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 392)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 392) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[50] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 400)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 400) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[51] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 408)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 408) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[52] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 416)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 416) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[53] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 424)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 424) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[54] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 432)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 432) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[55] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 440)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 440) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[56] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 448)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 448) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[57] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 456)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 456) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[58] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 464)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 464) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[59] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 472)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 472) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[60] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 480)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 480) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[61] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 488)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 488) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[62] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 496)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 496) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[63] * B */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 504)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 504) = (word64)(r8); + WC_S64(rdi, 512) = (word64)(r9); +} + +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_4096_cond_sub_64(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, r8, r9, rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[64]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 512; + rsp = (word64)(rsp - 512); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rdx, 136)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 128) = (word64)(r8); + WC_S64(rsp, 136) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 144)); + r9 = (word64)(WC_L64(rdx, 152)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 144) = (word64)(r8); + WC_S64(rsp, 152) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 160)); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 160) = (word64)(r8); + WC_S64(rsp, 168) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rdx, 184)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 176) = (word64)(r8); + WC_S64(rsp, 184) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 192)); + r9 = (word64)(WC_L64(rdx, 200)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 192) = (word64)(r8); + WC_S64(rsp, 200) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 208)); + r9 = (word64)(WC_L64(rdx, 216)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 208) = (word64)(r8); + WC_S64(rsp, 216) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 224)); + r9 = (word64)(WC_L64(rdx, 232)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 224) = (word64)(r8); + WC_S64(rsp, 232) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 240)); + r9 = (word64)(WC_L64(rdx, 248)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 240) = (word64)(r8); + WC_S64(rsp, 248) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 256)); + r9 = (word64)(WC_L64(rdx, 264)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 256) = (word64)(r8); + WC_S64(rsp, 264) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 272)); + r9 = (word64)(WC_L64(rdx, 280)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 272) = (word64)(r8); + WC_S64(rsp, 280) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 288)); + r9 = (word64)(WC_L64(rdx, 296)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 288) = (word64)(r8); + WC_S64(rsp, 296) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 304)); + r9 = (word64)(WC_L64(rdx, 312)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 304) = (word64)(r8); + WC_S64(rsp, 312) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 320)); + r9 = (word64)(WC_L64(rdx, 328)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 320) = (word64)(r8); + WC_S64(rsp, 328) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 336)); + r9 = (word64)(WC_L64(rdx, 344)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 336) = (word64)(r8); + WC_S64(rsp, 344) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 352)); + r9 = (word64)(WC_L64(rdx, 360)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 352) = (word64)(r8); + WC_S64(rsp, 360) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 368)); + r9 = (word64)(WC_L64(rdx, 376)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 368) = (word64)(r8); + WC_S64(rsp, 376) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 384)); + r9 = (word64)(WC_L64(rdx, 392)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 384) = (word64)(r8); + WC_S64(rsp, 392) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 400)); + r9 = (word64)(WC_L64(rdx, 408)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 400) = (word64)(r8); + WC_S64(rsp, 408) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 416)); + r9 = (word64)(WC_L64(rdx, 424)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 416) = (word64)(r8); + WC_S64(rsp, 424) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 432)); + r9 = (word64)(WC_L64(rdx, 440)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 432) = (word64)(r8); + WC_S64(rsp, 440) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 448)); + r9 = (word64)(WC_L64(rdx, 456)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 448) = (word64)(r8); + WC_S64(rsp, 456) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 464)); + r9 = (word64)(WC_L64(rdx, 472)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 464) = (word64)(r8); + WC_S64(rsp, 472) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 480)); + r9 = (word64)(WC_L64(rdx, 488)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 480) = (word64)(r8); + WC_S64(rsp, 488) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 496)); + r9 = (word64)(WC_L64(rdx, 504)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 496) = (word64)(r8); + WC_S64(rsp, 504) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _subborrow_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsp, 72)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 80)); + rdx = (word64)(WC_L64(rsp, 80)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsp, 88)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 96)); + rdx = (word64)(WC_L64(rsp, 96)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsp, 104)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 112)); + rdx = (word64)(WC_L64(rsp, 112)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsp, 120)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 128)); + rdx = (word64)(WC_L64(rsp, 128)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 136)); + rdx = (word64)(WC_L64(rsp, 136)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 144)); + rdx = (word64)(WC_L64(rsp, 144)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 136) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 152)); + rdx = (word64)(WC_L64(rsp, 152)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 144) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 160)); + rdx = (word64)(WC_L64(rsp, 160)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 152) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 168)); + rdx = (word64)(WC_L64(rsp, 168)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 160) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 176)); + rdx = (word64)(WC_L64(rsp, 176)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 184)); + rdx = (word64)(WC_L64(rsp, 184)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 192)); + rdx = (word64)(WC_L64(rsp, 192)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 184) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 200)); + rdx = (word64)(WC_L64(rsp, 200)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 192) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 208)); + rdx = (word64)(WC_L64(rsp, 208)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 200) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 216)); + rdx = (word64)(WC_L64(rsp, 216)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 208) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 224)); + rdx = (word64)(WC_L64(rsp, 224)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 216) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 232)); + rdx = (word64)(WC_L64(rsp, 232)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 224) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 240)); + rdx = (word64)(WC_L64(rsp, 240)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 232) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 248)); + rdx = (word64)(WC_L64(rsp, 248)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 240) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 256)); + rdx = (word64)(WC_L64(rsp, 256)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 248) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 264)); + rdx = (word64)(WC_L64(rsp, 264)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 256) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 272)); + rdx = (word64)(WC_L64(rsp, 272)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 264) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 280)); + rdx = (word64)(WC_L64(rsp, 280)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 272) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 288)); + rdx = (word64)(WC_L64(rsp, 288)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 280) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 296)); + rdx = (word64)(WC_L64(rsp, 296)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 288) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 304)); + rdx = (word64)(WC_L64(rsp, 304)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 296) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 312)); + rdx = (word64)(WC_L64(rsp, 312)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 304) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 320)); + rdx = (word64)(WC_L64(rsp, 320)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 312) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 328)); + rdx = (word64)(WC_L64(rsp, 328)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 320) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 336)); + rdx = (word64)(WC_L64(rsp, 336)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 328) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 344)); + rdx = (word64)(WC_L64(rsp, 344)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 336) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 352)); + rdx = (word64)(WC_L64(rsp, 352)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 344) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 360)); + rdx = (word64)(WC_L64(rsp, 360)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 352) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 368)); + rdx = (word64)(WC_L64(rsp, 368)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 360) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 376)); + rdx = (word64)(WC_L64(rsp, 376)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 368) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 384)); + rdx = (word64)(WC_L64(rsp, 384)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 376) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 392)); + rdx = (word64)(WC_L64(rsp, 392)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 384) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 400)); + rdx = (word64)(WC_L64(rsp, 400)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 392) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 408)); + rdx = (word64)(WC_L64(rsp, 408)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 400) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 416)); + rdx = (word64)(WC_L64(rsp, 416)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 408) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 424)); + rdx = (word64)(WC_L64(rsp, 424)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 416) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 432)); + rdx = (word64)(WC_L64(rsp, 432)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 424) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 440)); + rdx = (word64)(WC_L64(rsp, 440)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 432) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 448)); + rdx = (word64)(WC_L64(rsp, 448)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 440) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 456)); + rdx = (word64)(WC_L64(rsp, 456)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 448) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 464)); + rdx = (word64)(WC_L64(rsp, 464)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 456) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 472)); + rdx = (word64)(WC_L64(rsp, 472)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 464) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 480)); + rdx = (word64)(WC_L64(rsp, 480)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 472) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 488)); + rdx = (word64)(WC_L64(rsp, 488)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 480) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 496)); + rdx = (word64)(WC_L64(rsp, 496)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 488) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 504)); + rdx = (word64)(WC_L64(rsp, 504)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 496) = (word64)(r8); + WC_S64(rdi, 504) = (word64)(r9); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Reduce the number back to 4096 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_4096_mont_reduce_64(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rsi, rcx, r15, r8, r13, r14, r11 = 0, rax = 0, r10 = 0, + rdx = 0, r9 = 0, r12 = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)mp; + + r15 = (word64)(0); + /* i = 64 */ + r8 = (word64)(0x40); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 8)); +L_4096_mont_reduce_64_loop: + /* mu = a[i] * mp */ + r11 = (word64)(r13); + r11 = (word64)(r11 * rcx); + /* a[i+0] += m[0] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* a[i+1] += m[1] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r13, r10, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+2] += m[2] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+3] += m[3] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 24) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+4] += m[4] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rdi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+5] += m[5] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 40) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+6] += m[6] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rdi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 48) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+7] += m[7] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r12 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 56) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+8] += m[8] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r12 = (word64)(WC_L64(rdi, 64)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 64) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+9] += m[9] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r12 = (word64)(WC_L64(rdi, 72)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 72) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+10] += m[10] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r12 = (word64)(WC_L64(rdi, 80)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 80) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+11] += m[11] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r12 = (word64)(WC_L64(rdi, 88)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 88) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+12] += m[12] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r12 = (word64)(WC_L64(rdi, 96)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 96) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+13] += m[13] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + r12 = (word64)(WC_L64(rdi, 104)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 104) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+14] += m[14] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r12 = (word64)(WC_L64(rdi, 112)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 112) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+15] += m[15] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + r12 = (word64)(WC_L64(rdi, 120)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 120) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+16] += m[16] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 128)); + r12 = (word64)(WC_L64(rdi, 128)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 128) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+17] += m[17] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 136)); + r12 = (word64)(WC_L64(rdi, 136)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 136) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+18] += m[18] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 144)); + r12 = (word64)(WC_L64(rdi, 144)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 144) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+19] += m[19] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 152)); + r12 = (word64)(WC_L64(rdi, 152)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 152) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+20] += m[20] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 160)); + r12 = (word64)(WC_L64(rdi, 160)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 160) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+21] += m[21] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 168)); + r12 = (word64)(WC_L64(rdi, 168)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 168) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+22] += m[22] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 176)); + r12 = (word64)(WC_L64(rdi, 176)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 176) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+23] += m[23] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 184)); + r12 = (word64)(WC_L64(rdi, 184)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 184) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+24] += m[24] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 192)); + r12 = (word64)(WC_L64(rdi, 192)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 192) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+25] += m[25] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 200)); + r12 = (word64)(WC_L64(rdi, 200)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 200) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+26] += m[26] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 208)); + r12 = (word64)(WC_L64(rdi, 208)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 208) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+27] += m[27] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 216)); + r12 = (word64)(WC_L64(rdi, 216)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 216) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+28] += m[28] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 224)); + r12 = (word64)(WC_L64(rdi, 224)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 224) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+29] += m[29] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 232)); + r12 = (word64)(WC_L64(rdi, 232)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 232) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+30] += m[30] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 240)); + r12 = (word64)(WC_L64(rdi, 240)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 240) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+31] += m[31] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 248)); + r12 = (word64)(WC_L64(rdi, 248)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 248) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+32] += m[32] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 256)); + r12 = (word64)(WC_L64(rdi, 256)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 256) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+33] += m[33] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 264)); + r12 = (word64)(WC_L64(rdi, 264)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 264) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+34] += m[34] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 272)); + r12 = (word64)(WC_L64(rdi, 272)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 272) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+35] += m[35] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 280)); + r12 = (word64)(WC_L64(rdi, 280)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 280) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+36] += m[36] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 288)); + r12 = (word64)(WC_L64(rdi, 288)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 288) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+37] += m[37] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 296)); + r12 = (word64)(WC_L64(rdi, 296)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 296) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+38] += m[38] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 304)); + r12 = (word64)(WC_L64(rdi, 304)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 304) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+39] += m[39] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 312)); + r12 = (word64)(WC_L64(rdi, 312)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 312) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+40] += m[40] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 320)); + r12 = (word64)(WC_L64(rdi, 320)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 320) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+41] += m[41] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 328)); + r12 = (word64)(WC_L64(rdi, 328)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 328) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+42] += m[42] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 336)); + r12 = (word64)(WC_L64(rdi, 336)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 336) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+43] += m[43] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 344)); + r12 = (word64)(WC_L64(rdi, 344)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 344) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+44] += m[44] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 352)); + r12 = (word64)(WC_L64(rdi, 352)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 352) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+45] += m[45] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 360)); + r12 = (word64)(WC_L64(rdi, 360)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 360) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+46] += m[46] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 368)); + r12 = (word64)(WC_L64(rdi, 368)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 368) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+47] += m[47] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 376)); + r12 = (word64)(WC_L64(rdi, 376)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 376) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+48] += m[48] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 384)); + r12 = (word64)(WC_L64(rdi, 384)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 384) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+49] += m[49] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 392)); + r12 = (word64)(WC_L64(rdi, 392)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 392) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+50] += m[50] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 400)); + r12 = (word64)(WC_L64(rdi, 400)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 400) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+51] += m[51] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 408)); + r12 = (word64)(WC_L64(rdi, 408)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 408) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+52] += m[52] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 416)); + r12 = (word64)(WC_L64(rdi, 416)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 416) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+53] += m[53] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 424)); + r12 = (word64)(WC_L64(rdi, 424)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 424) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+54] += m[54] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 432)); + r12 = (word64)(WC_L64(rdi, 432)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 432) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+55] += m[55] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 440)); + r12 = (word64)(WC_L64(rdi, 440)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 440) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+56] += m[56] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 448)); + r12 = (word64)(WC_L64(rdi, 448)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 448) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+57] += m[57] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 456)); + r12 = (word64)(WC_L64(rdi, 456)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 456) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+58] += m[58] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 464)); + r12 = (word64)(WC_L64(rdi, 464)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 464) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+59] += m[59] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 472)); + r12 = (word64)(WC_L64(rdi, 472)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 472) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+60] += m[60] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 480)); + r12 = (word64)(WC_L64(rdi, 480)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 480) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+61] += m[61] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 488)); + r12 = (word64)(WC_L64(rdi, 488)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 488) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+62] += m[62] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 496)); + r12 = (word64)(WC_L64(rdi, 496)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 496) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+63] += m[63] * mu */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 504)); + r12 = (word64)(WC_L64(rdi, 504)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, r15, (unsigned long long*)&rdx); + r15 = (word64)(0); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 504) = (word64)(r12); + cf = _addcarry_u64(cf, WC_L64(rdi, 512), rdx, (unsigned long long*)&WC_S64( + rdi, 512)); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* i -= 1 */ + rdi = (word64)(rdi + 8); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_4096_mont_reduce_64_loop; + } + WC_S64(rdi, 0) = (word64)(r13); + WC_S64(rdi, 8) = (word64)(r14); + r15 = (word64)(0 - r15); +#ifdef _WIN64 + rdx = (word64)(rsi); + rcx = (word64)(r15); +#else + rcx = (word64)(r15); + rdx = (word64)(rsi); +#endif /* _WIN64 */ + rsi = (word64)(rdi); + rdi = (word64)(rdi); + rdi = (word64)(rdi - 0x200); + (void)sp_4096_cond_sub_64((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx, (sp_digit)rcx); +} + +/* Sub b from a into r. (r = a - b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_4096_sub_64(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, r8, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rcx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 96)); + WC_S64(rdi, 88) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 96), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 104)); + WC_S64(rdi, 96) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 112)); + WC_S64(rdi, 104) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 112), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 120)); + WC_S64(rdi, 112) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 120), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 128)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 128), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 136)); + WC_S64(rdi, 128) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 136), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 144)); + WC_S64(rdi, 136) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 144), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 152)); + WC_S64(rdi, 144) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 152), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 160)); + WC_S64(rdi, 152) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 160), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 168)); + WC_S64(rdi, 160) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 168), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 176)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 176), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 184)); + WC_S64(rdi, 176) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 184), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 192)); + WC_S64(rdi, 184) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 192), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 200)); + WC_S64(rdi, 192) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 200), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 208)); + WC_S64(rdi, 200) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 208), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 216)); + WC_S64(rdi, 208) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 216), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 224)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 224), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 232)); + WC_S64(rdi, 224) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 232), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 240)); + WC_S64(rdi, 232) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 240), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 248)); + WC_S64(rdi, 240) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 248), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 256)); + WC_S64(rdi, 248) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 256), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 264)); + WC_S64(rdi, 256) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 264), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 272)); + WC_S64(rdi, 264) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 272), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 280)); + WC_S64(rdi, 272) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 280), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 288)); + WC_S64(rdi, 280) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 288), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 296)); + WC_S64(rdi, 288) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 296), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 304)); + WC_S64(rdi, 296) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 304), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 312)); + WC_S64(rdi, 304) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 312), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 320)); + WC_S64(rdi, 312) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 320), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 328)); + WC_S64(rdi, 320) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 328), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 336)); + WC_S64(rdi, 328) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 336), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 344)); + WC_S64(rdi, 336) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 344), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 352)); + WC_S64(rdi, 344) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 352), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 360)); + WC_S64(rdi, 352) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 360), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 368)); + WC_S64(rdi, 360) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 368), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 376)); + WC_S64(rdi, 368) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 376), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 384)); + WC_S64(rdi, 376) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 384), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 392)); + WC_S64(rdi, 384) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 392), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 400)); + WC_S64(rdi, 392) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 400), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 408)); + WC_S64(rdi, 400) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 408), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 416)); + WC_S64(rdi, 408) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 416), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 424)); + WC_S64(rdi, 416) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 424), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 432)); + WC_S64(rdi, 424) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 432), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 440)); + WC_S64(rdi, 432) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 440), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 448)); + WC_S64(rdi, 440) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 448), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 456)); + WC_S64(rdi, 448) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 456), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 464)); + WC_S64(rdi, 456) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 464), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 472)); + WC_S64(rdi, 464) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 472), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 480)); + WC_S64(rdi, 472) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 480), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 488)); + WC_S64(rdi, 480) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 488), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 496)); + WC_S64(rdi, 488) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 496), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 504)); + WC_S64(rdi, 496) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 504), (unsigned long long*)&r8); + WC_S64(rdi, 504) = (word64)(r8); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_4096_mul_d_avx2_64(sp_digit* r, const sp_digit* a, + const sp_digit b) +{ + sp_4096_mul_d_64(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef _WIN64 +/* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) + * + * @param [in] d1 The high order half of the number to divide. + * @param [in] d0 The low order half of the number to divide. + * @param [in] div The dividend. + * + * @return The result of the division. + */ +WOLFSSL_LOCAL sp_digit div_4096_word_asm_64(sp_digit d1, sp_digit d0, + sp_digit div) +{ + word64 rdi, rsi, rcx, rax, rdx; + + rdi = (word64)(word64)d1; + rsi = (word64)(word64)d0; + rcx = (word64)(word64)div; + + rax = (word64)(rsi); + rdx = (word64)(rdi); + WC_X64I_DIV128(rax, rdx, rdx, rax, rcx); + return (sp_digit)(word64)rax; +} + +#endif /* _WIN64 */ +#ifdef HAVE_INTEL_AVX2 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_4096_cond_sub_avx2_64(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r10, r8, r9, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _subborrow_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 64) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 80) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 88) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 104)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 96) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 104) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 112) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rsi, 128)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 128) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 144)); + r8 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 136) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 152)); + r9 = (word64)(WC_L64(rsi, 152)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 144) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 152) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 160) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rsi, 176)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 176) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 192)); + r8 = (word64)(WC_L64(rsi, 192)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 184) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 200)); + r9 = (word64)(WC_L64(rsi, 200)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 192) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 208)); + r10 = (word64)(WC_L64(rsi, 208)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 200) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 216)); + r8 = (word64)(WC_L64(rsi, 216)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 208) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 224)); + r9 = (word64)(WC_L64(rsi, 224)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 232)); + r10 = (word64)(WC_L64(rsi, 232)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 224) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 240)); + r8 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 232) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 248)); + r9 = (word64)(WC_L64(rsi, 248)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 240) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 256)); + r10 = (word64)(WC_L64(rsi, 256)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 248) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 264)); + r8 = (word64)(WC_L64(rsi, 264)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 256) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 272)); + r9 = (word64)(WC_L64(rsi, 272)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 264) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 280)); + r10 = (word64)(WC_L64(rsi, 280)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 272) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 288)); + r8 = (word64)(WC_L64(rsi, 288)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 280) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 296)); + r9 = (word64)(WC_L64(rsi, 296)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 288) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 304)); + r10 = (word64)(WC_L64(rsi, 304)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 296) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 312)); + r8 = (word64)(WC_L64(rsi, 312)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 304) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 320)); + r9 = (word64)(WC_L64(rsi, 320)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 312) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 328)); + r10 = (word64)(WC_L64(rsi, 328)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 320) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 336)); + r8 = (word64)(WC_L64(rsi, 336)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 328) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 344)); + r9 = (word64)(WC_L64(rsi, 344)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 336) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 352)); + r10 = (word64)(WC_L64(rsi, 352)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 344) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 360)); + r8 = (word64)(WC_L64(rsi, 360)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 352) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 368)); + r9 = (word64)(WC_L64(rsi, 368)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 360) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 376)); + r10 = (word64)(WC_L64(rsi, 376)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 368) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 384)); + r8 = (word64)(WC_L64(rsi, 384)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 376) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 392)); + r9 = (word64)(WC_L64(rsi, 392)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 384) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 400)); + r10 = (word64)(WC_L64(rsi, 400)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 392) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 408)); + r8 = (word64)(WC_L64(rsi, 408)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 400) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 416)); + r9 = (word64)(WC_L64(rsi, 416)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 408) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 424)); + r10 = (word64)(WC_L64(rsi, 424)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 416) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 432)); + r8 = (word64)(WC_L64(rsi, 432)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 424) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 440)); + r9 = (word64)(WC_L64(rsi, 440)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 432) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 448)); + r10 = (word64)(WC_L64(rsi, 448)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 440) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 456)); + r8 = (word64)(WC_L64(rsi, 456)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 448) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 464)); + r9 = (word64)(WC_L64(rsi, 464)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 456) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 472)); + r10 = (word64)(WC_L64(rsi, 472)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 464) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 480)); + r8 = (word64)(WC_L64(rsi, 480)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 472) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 488)); + r9 = (word64)(WC_L64(rsi, 488)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 480) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 496)); + r10 = (word64)(WC_L64(rsi, 496)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 488) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 504)); + r8 = (word64)(WC_L64(rsi, 504)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 496) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + WC_S64(rdi, 504) = (word64)(r8); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Compare a with b in constant time. + * + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + * + * @return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +WOLFSSL_LOCAL sp_int64 sp_4096_cmp_64(const sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + word64 zf7; + word64 zf8; + word64 zf9; + word64 zf10; + word64 zf11; + word64 zf12; + word64 zf13; + word64 zf14; + word64 zf15; + word64 zf16; + word64 zf17; + word64 zf18; + word64 zf19; + word64 zf20; + word64 zf21; + word64 zf22; + word64 zf23; + word64 zf24; + word64 zf25; + word64 zf26; + word64 zf27; + word64 zf28; + word64 zf29; + word64 zf30; + word64 zf31; + word64 zf32; + word64 zf33; + word64 zf34; + word64 zf35; + word64 zf36; + word64 zf37; + word64 zf38; + word64 zf39; + word64 zf40; + word64 zf41; + word64 zf42; + word64 zf43; + word64 zf44; + word64 zf45; + word64 zf46; + word64 zf47; + word64 zf48; + word64 zf49; + word64 zf50; + word64 zf51; + word64 zf52; + word64 zf53; + word64 zf54; + word64 zf55; + word64 zf56; + word64 zf57; + word64 zf58; + word64 zf59; + word64 zf60; + word64 zf61; + word64 zf62; + word64 zf63; + word64 zf64; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rcx = (word64)(0); + rdx = (word64)(-1); + rax = (word64)(-1); + r8 = (word64)(1); + r9 = (word64)(WC_L64(rdi, 504)); + r10 = (word64)(WC_L64(rsi, 504)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf1 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf1) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 496)); + r10 = (word64)(WC_L64(rsi, 496)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf2 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf2) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 488)); + r10 = (word64)(WC_L64(rsi, 488)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf3 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf3) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 480)); + r10 = (word64)(WC_L64(rsi, 480)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf4 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf4) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 472)); + r10 = (word64)(WC_L64(rsi, 472)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf5 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf5) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 464)); + r10 = (word64)(WC_L64(rsi, 464)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf6 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf6) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 456)); + r10 = (word64)(WC_L64(rsi, 456)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf7 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf7) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 448)); + r10 = (word64)(WC_L64(rsi, 448)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf8 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf8) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 440)); + r10 = (word64)(WC_L64(rsi, 440)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf9 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf9) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 432)); + r10 = (word64)(WC_L64(rsi, 432)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf10 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf10) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 424)); + r10 = (word64)(WC_L64(rsi, 424)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf11 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf11) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 416)); + r10 = (word64)(WC_L64(rsi, 416)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf12 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf12) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 408)); + r10 = (word64)(WC_L64(rsi, 408)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf13 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf13) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 400)); + r10 = (word64)(WC_L64(rsi, 400)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf14 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf14) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 392)); + r10 = (word64)(WC_L64(rsi, 392)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf15 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf15) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 384)); + r10 = (word64)(WC_L64(rsi, 384)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf16 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf16) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 376)); + r10 = (word64)(WC_L64(rsi, 376)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf17 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf17) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 368)); + r10 = (word64)(WC_L64(rsi, 368)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf18 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf18) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 360)); + r10 = (word64)(WC_L64(rsi, 360)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf19 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf19) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 352)); + r10 = (word64)(WC_L64(rsi, 352)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf20 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf20) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 344)); + r10 = (word64)(WC_L64(rsi, 344)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf21 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf21) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 336)); + r10 = (word64)(WC_L64(rsi, 336)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf22 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf22) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 328)); + r10 = (word64)(WC_L64(rsi, 328)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf23 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf23) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 320)); + r10 = (word64)(WC_L64(rsi, 320)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf24 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf24) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 312)); + r10 = (word64)(WC_L64(rsi, 312)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf25 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf25) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 304)); + r10 = (word64)(WC_L64(rsi, 304)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf26 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf26) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 296)); + r10 = (word64)(WC_L64(rsi, 296)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf27 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf27) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 288)); + r10 = (word64)(WC_L64(rsi, 288)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf28 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf28) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 280)); + r10 = (word64)(WC_L64(rsi, 280)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf29 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf29) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 272)); + r10 = (word64)(WC_L64(rsi, 272)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf30 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf30) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 264)); + r10 = (word64)(WC_L64(rsi, 264)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf31 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf31) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 256)); + r10 = (word64)(WC_L64(rsi, 256)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf32 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf32) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 248)); + r10 = (word64)(WC_L64(rsi, 248)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf33 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf33) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 240)); + r10 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf34 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf34) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 232)); + r10 = (word64)(WC_L64(rsi, 232)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf35 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf35) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 224)); + r10 = (word64)(WC_L64(rsi, 224)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf36 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf36) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 216)); + r10 = (word64)(WC_L64(rsi, 216)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf37 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf37) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 208)); + r10 = (word64)(WC_L64(rsi, 208)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf38 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf38) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 200)); + r10 = (word64)(WC_L64(rsi, 200)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf39 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf39) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 192)); + r10 = (word64)(WC_L64(rsi, 192)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf40 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf40) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf41 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf41) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 176)); + r10 = (word64)(WC_L64(rsi, 176)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf42 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf42) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 168)); + r10 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf43 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf43) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf44 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf44) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 152)); + r10 = (word64)(WC_L64(rsi, 152)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf45 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf45) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 144)); + r10 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf46 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf46) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf47 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf47) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 128)); + r10 = (word64)(WC_L64(rsi, 128)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf48 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf48) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 120)); + r10 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf49 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf49) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf50 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf50) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 104)); + r10 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf51 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf51) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf52 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf52) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf53 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf53) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf54 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf54) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf55 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf55) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf56 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf56) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(WC_L64(rsi, 56)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf57 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf57) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf58 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf58) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf59 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf59) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf60 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf60) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf61 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf61) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf62 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf62) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf63 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf63) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf64 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf64) != (0) ? rcx : rdx; + rax = (word64)(rax ^ rdx); + return (sp_int64)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 4096 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_4096_mont_reduce_avx2_64(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + sp_4096_mont_reduce_64(a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifndef WC_NO_CACHE_RESISTANT +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_4096_get_from_table_avx2_64(sp_digit* r, sp_digit** table, + int idx) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y10, y11, y12, y13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y10 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rdx)); + y11 = _mm256_zextsi128_si256(_mm_cvtsi64_si128((long long)rax)); + y13 = _mm256_setzero_si256(); + y10 = _mm256_permutevar8x32_epi32(y10, y13); + y11 = _mm256_permutevar8x32_epi32(y11, y13); + /* START: 0-15 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + rdi = (word64)(rdi + 0x80); + /* END: 0-15 */ + /* START: 16-31 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + rcx = (word64)(rcx + 0x80); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + rdi = (word64)(rdi + 0x80); + /* END: 16-31 */ + /* START: 32-47 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + rcx = (word64)(rcx + 0x100); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + rdi = (word64)(rdi + 0x80); + /* END: 32-47 */ + /* START: 48-63 */ + y13 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + /* ENTRY: 0 */ + rcx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 1 */ + rcx = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 2 */ + rcx = (word64)(WC_L64(rsi, 16)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 3 */ + rcx = (word64)(WC_L64(rsi, 24)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 4 */ + rcx = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 5 */ + rcx = (word64)(WC_L64(rsi, 40)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 6 */ + rcx = (word64)(WC_L64(rsi, 48)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 7 */ + rcx = (word64)(WC_L64(rsi, 56)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 8 */ + rcx = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 9 */ + rcx = (word64)(WC_L64(rsi, 72)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 10 */ + rcx = (word64)(WC_L64(rsi, 80)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 11 */ + rcx = (word64)(WC_L64(rsi, 88)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 12 */ + rcx = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 13 */ + rcx = (word64)(WC_L64(rsi, 104)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 14 */ + rcx = (word64)(WC_L64(rsi, 112)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + /* ENTRY: 15 */ + rcx = (word64)(WC_L64(rsi, 120)); + rcx = (word64)(rcx + 0x180); + y12 = _mm256_cmpeq_epi32(y13, y10); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y6 = _mm256_or_si256(y6, y2); + y7 = _mm256_or_si256(y7, y3); + y13 = _mm256_add_epi32(y13, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y7); + /* END: 48-63 */ +} + +#endif /* !WC_NO_CACHE_RESISTANT */ +/* Conditionally add a and b using the mask m. + * m is -1 to add and 0 when not. + * + * @param [out] r A single precision number representing conditional add + * result. + * @param [in] a A single precision number to add with. + * @param [in] b A single precision number to add. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_4096_cond_add_32(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, rax, r8, r9; + XALIGNED(32) WC_X64I_SLOT stk[32]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 256; + rsp = (word64)(rsp - 256); + rax = (word64)(0); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rdx, 136)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 128) = (word64)(r8); + WC_S64(rsp, 136) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 144)); + r9 = (word64)(WC_L64(rdx, 152)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 144) = (word64)(r8); + WC_S64(rsp, 152) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 160)); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 160) = (word64)(r8); + WC_S64(rsp, 168) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rdx, 184)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 176) = (word64)(r8); + WC_S64(rsp, 184) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 192)); + r9 = (word64)(WC_L64(rdx, 200)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 192) = (word64)(r8); + WC_S64(rsp, 200) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 208)); + r9 = (word64)(WC_L64(rdx, 216)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 208) = (word64)(r8); + WC_S64(rsp, 216) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 224)); + r9 = (word64)(WC_L64(rdx, 232)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 224) = (word64)(r8); + WC_S64(rsp, 232) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 240)); + r9 = (word64)(WC_L64(rdx, 248)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 240) = (word64)(r8); + WC_S64(rsp, 248) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _addcarry_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsp, 72)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 80)); + rdx = (word64)(WC_L64(rsp, 80)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsp, 88)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 96)); + rdx = (word64)(WC_L64(rsp, 96)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsp, 104)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 112)); + rdx = (word64)(WC_L64(rsp, 112)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsp, 120)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 128)); + rdx = (word64)(WC_L64(rsp, 128)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 136)); + rdx = (word64)(WC_L64(rsp, 136)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 144)); + rdx = (word64)(WC_L64(rsp, 144)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 136) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 152)); + rdx = (word64)(WC_L64(rsp, 152)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 144) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 160)); + rdx = (word64)(WC_L64(rsp, 160)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 152) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 168)); + rdx = (word64)(WC_L64(rsp, 168)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 160) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 176)); + rdx = (word64)(WC_L64(rsp, 176)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 168) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 184)); + rdx = (word64)(WC_L64(rsp, 184)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 176) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 192)); + rdx = (word64)(WC_L64(rsp, 192)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 184) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 200)); + rdx = (word64)(WC_L64(rsp, 200)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 192) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 208)); + rdx = (word64)(WC_L64(rsp, 208)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 200) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 216)); + rdx = (word64)(WC_L64(rsp, 216)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 208) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 224)); + rdx = (word64)(WC_L64(rsp, 224)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 216) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 232)); + rdx = (word64)(WC_L64(rsp, 232)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 224) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 240)); + rdx = (word64)(WC_L64(rsp, 240)); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 232) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 248)); + rdx = (word64)(WC_L64(rsp, 248)); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 240) = (word64)(r8); + WC_S64(rdi, 248) = (word64)(r9); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Conditionally add a and b using the mask m. + * m is -1 to add and 0 when not. + * + * @param [out] r A single precision number representing conditional add + * result. + * @param [in] a A single precision number to add with. + * @param [in] b A single precision number to add. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_4096_cond_add_avx2_32(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rax = (word64)(0); + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 64) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 80) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 88) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 104)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 96) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 104) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 112) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 128)); + r9 = (word64)(WC_L64(rsi, 128)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 136)); + r10 = (word64)(WC_L64(rsi, 136)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 128) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 144)); + r8 = (word64)(WC_L64(rsi, 144)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 136) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 152)); + r9 = (word64)(WC_L64(rsi, 152)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 144) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 160)); + r10 = (word64)(WC_L64(rsi, 160)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 152) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 168)); + r8 = (word64)(WC_L64(rsi, 168)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 160) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 176)); + r9 = (word64)(WC_L64(rsi, 176)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 168) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 184)); + r10 = (word64)(WC_L64(rsi, 184)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 176) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 192)); + r8 = (word64)(WC_L64(rsi, 192)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 184) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 200)); + r9 = (word64)(WC_L64(rsi, 200)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 192) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 208)); + r10 = (word64)(WC_L64(rsi, 208)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 200) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 216)); + r8 = (word64)(WC_L64(rsi, 216)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 208) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 224)); + r9 = (word64)(WC_L64(rsi, 224)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 216) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 232)); + r10 = (word64)(WC_L64(rsi, 232)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 224) = (word64)(r9); + cf = _addcarry_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 240)); + r8 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 232) = (word64)(r10); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 248)); + r9 = (word64)(WC_L64(rsi, 248)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 240) = (word64)(r8); + cf = _addcarry_u64(cf, r9, r10, (unsigned long long*)&r9); + WC_S64(rdi, 248) = (word64)(r9); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Shift number left by n bit. (r = a << n) + * + * @param [out] r Result of left shift by n. + * @param [in] a Number to shift. + * @param [in] n Amoutnt o shift. + */ +WOLFSSL_LOCAL void sp_4096_lshift_64(sp_digit* r, const sp_digit* a, int n) +{ + word64 rdi, rsi, rcx, r10, r11, rdx, rax, r8, r9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r10 = (word64)(0); + r11 = (word64)(WC_L64(rsi, 472)); + rdx = (word64)(WC_L64(rsi, 480)); + rax = (word64)(WC_L64(rsi, 488)); + r8 = (word64)(WC_L64(rsi, 496)); + r9 = (word64)(WC_L64(rsi, 504)); + r10 = (word64)((r10 << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 480) = (word64)(rdx); + WC_S64(rdi, 488) = (word64)(rax); + WC_S64(rdi, 496) = (word64)(r8); + WC_S64(rdi, 504) = (word64)(r9); + WC_S64(rdi, 512) = (word64)(r10); + r9 = (word64)(WC_L64(rsi, 440)); + rdx = (word64)(WC_L64(rsi, 448)); + rax = (word64)(WC_L64(rsi, 456)); + r8 = (word64)(WC_L64(rsi, 464)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 448) = (word64)(rdx); + WC_S64(rdi, 456) = (word64)(rax); + WC_S64(rdi, 464) = (word64)(r8); + WC_S64(rdi, 472) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 408)); + rdx = (word64)(WC_L64(rsi, 416)); + rax = (word64)(WC_L64(rsi, 424)); + r8 = (word64)(WC_L64(rsi, 432)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 416) = (word64)(rdx); + WC_S64(rdi, 424) = (word64)(rax); + WC_S64(rdi, 432) = (word64)(r8); + WC_S64(rdi, 440) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 376)); + rdx = (word64)(WC_L64(rsi, 384)); + rax = (word64)(WC_L64(rsi, 392)); + r8 = (word64)(WC_L64(rsi, 400)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 384) = (word64)(rdx); + WC_S64(rdi, 392) = (word64)(rax); + WC_S64(rdi, 400) = (word64)(r8); + WC_S64(rdi, 408) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 344)); + rdx = (word64)(WC_L64(rsi, 352)); + rax = (word64)(WC_L64(rsi, 360)); + r8 = (word64)(WC_L64(rsi, 368)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 352) = (word64)(rdx); + WC_S64(rdi, 360) = (word64)(rax); + WC_S64(rdi, 368) = (word64)(r8); + WC_S64(rdi, 376) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 312)); + rdx = (word64)(WC_L64(rsi, 320)); + rax = (word64)(WC_L64(rsi, 328)); + r8 = (word64)(WC_L64(rsi, 336)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 320) = (word64)(rdx); + WC_S64(rdi, 328) = (word64)(rax); + WC_S64(rdi, 336) = (word64)(r8); + WC_S64(rdi, 344) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 280)); + rdx = (word64)(WC_L64(rsi, 288)); + rax = (word64)(WC_L64(rsi, 296)); + r8 = (word64)(WC_L64(rsi, 304)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 288) = (word64)(rdx); + WC_S64(rdi, 296) = (word64)(rax); + WC_S64(rdi, 304) = (word64)(r8); + WC_S64(rdi, 312) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 248)); + rdx = (word64)(WC_L64(rsi, 256)); + rax = (word64)(WC_L64(rsi, 264)); + r8 = (word64)(WC_L64(rsi, 272)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 256) = (word64)(rdx); + WC_S64(rdi, 264) = (word64)(rax); + WC_S64(rdi, 272) = (word64)(r8); + WC_S64(rdi, 280) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 216)); + rdx = (word64)(WC_L64(rsi, 224)); + rax = (word64)(WC_L64(rsi, 232)); + r8 = (word64)(WC_L64(rsi, 240)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 224) = (word64)(rdx); + WC_S64(rdi, 232) = (word64)(rax); + WC_S64(rdi, 240) = (word64)(r8); + WC_S64(rdi, 248) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 184)); + rdx = (word64)(WC_L64(rsi, 192)); + rax = (word64)(WC_L64(rsi, 200)); + r8 = (word64)(WC_L64(rsi, 208)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 192) = (word64)(rdx); + WC_S64(rdi, 200) = (word64)(rax); + WC_S64(rdi, 208) = (word64)(r8); + WC_S64(rdi, 216) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 152)); + rdx = (word64)(WC_L64(rsi, 160)); + rax = (word64)(WC_L64(rsi, 168)); + r8 = (word64)(WC_L64(rsi, 176)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 160) = (word64)(rdx); + WC_S64(rdi, 168) = (word64)(rax); + WC_S64(rdi, 176) = (word64)(r8); + WC_S64(rdi, 184) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsi, 128)); + rax = (word64)(WC_L64(rsi, 136)); + r8 = (word64)(WC_L64(rsi, 144)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 128) = (word64)(rdx); + WC_S64(rdi, 136) = (word64)(rax); + WC_S64(rdi, 144) = (word64)(r8); + WC_S64(rdi, 152) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsi, 96)); + rax = (word64)(WC_L64(rsi, 104)); + r8 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 96) = (word64)(rdx); + WC_S64(rdi, 104) = (word64)(rax); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsi, 64)); + rax = (word64)(WC_L64(rsi, 72)); + r8 = (word64)(WC_L64(rsi, 80)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 64) = (word64)(rdx); + WC_S64(rdi, 72) = (word64)(rax); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsi, 32)); + rax = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 32) = (word64)(rdx); + WC_S64(rdi, 40) = (word64)(rax); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)(rdx << ((byte)rcx & 63)); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r11); +} + +#endif /* WOLFSSL_SP_4096 */ +#endif /* WOLFSSL_SP_4096 */ +#ifndef WOLFSSL_SP_NO_256 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_256_mul_4(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rcx, rsp, rax, rdx = 0, r10, r9, r8; + XALIGNED(32) WC_X64I_SLOT stk[4]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 32); + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 8) = (word64)(r9); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 16) = (word64)(r10); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 24) = (word64)(r8); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 32) = (word64)(r9); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 40) = (word64)(r10); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r8 = (word64)(WC_L64(rsp, 16)); + r9 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply. + * @param [in] b Second number to multiply. + */ +WOLFSSL_LOCAL void sp_256_mul_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + sp_256_mul_4(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_256_sqr_4(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rax, rdx = 0, r9, r8, rcx, r10, r11; + XALIGNED(32) WC_X64I_SLOT stk[4]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 32; + rsp = (word64)(rsp - 32); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 8) = (word64)(r8); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 16) = (word64)(r9); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 24) = (word64)(rcx); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 32) = (word64)(r8); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(rcx); + WC_S64(rdi, 56) = (word64)(r8); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r10 = (word64)(WC_L64(rsp, 16)); + r11 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); +} + +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * @param [out] r Result of squaring. + * @param [in] a Number to square in Montgomery form. + */ +WOLFSSL_LOCAL void sp_256_sqr_avx2_4(sp_digit* r, const sp_digit* a) +{ + sp_256_sqr_4(r, a); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_256_add_4(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(0); + rcx = (word64)(WC_L64(rsi, 0)); + r8 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(WC_L64(rsi, 16)); + r10 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Sub b from a into r. (r = a - b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_256_sub_4(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(0); + rcx = (word64)(WC_L64(rsi, 0)); + r8 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(WC_L64(rsi, 16)); + r10 = (word64)(WC_L64(rsi, 24)); + cf = _subborrow_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Conditionally copy a into r using the mask m. + * m is -1 to copy and 0 when not. + * + * @param [out] r A single precision number to copy over. + * @param [in] a A single precision number to copy. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL void sp_256_cond_copy_4(sp_digit* r, const sp_digit* a, + sp_digit m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(word64)m; + + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)(WC_L64(rdi, 16)); + r9 = (word64)(WC_L64(rdi, 24)); + rax = (word64)(rax ^ WC_L64(rsi, 0)); + rcx = (word64)(rcx ^ WC_L64(rsi, 8)); + r8 = (word64)(r8 ^ WC_L64(rsi, 16)); + r9 = (word64)(r9 ^ WC_L64(rsi, 24)); + rax = (word64)(rax & rdx); + rcx = (word64)(rcx & rdx); + r8 = (word64)(r8 & rdx); + r9 = (word64)(r9 & rdx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) ^ rax); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) ^ rcx); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) ^ r8); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) ^ r9); +} + +/* Multiply two Montgomery form numbers mod the modulus (prime). + * (r = a * b mod m) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply in Montgomery form. + * @param [in] b Second number to multiply in Montgomery form. + * @param [in] m Modulus (prime). + * @param [in] mp Montgomery multiplier. + */ +WOLFSSL_LOCAL void sp_256_mont_mul_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m, sp_digit mp) +{ + word64 rdi, rsi, rcx, r8, rax, rdx = 0, r9, r10, r11, r12, r13, r14, r15, + rbx; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(size_t)b; + r8 = (word64)(size_t)m; + + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r15 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rbx = (word64)(0); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + /* Start Reduction */ + /* mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192 */ + /* - a[0] << 32 << 192 */ + /* a[0]-a[3] + (a[0] * 2) << 192 */ + rax = (word64)(r9); + rdx = (word64)(r12 + r9 * 2); + rsi = (word64)(r10); + rcx = (word64)(r11); + r8 = (word64)(r11); + /* a[0]-a[2] << 32 */ + r9 = (word64)(r9 << 32); + r8 = (word64)((r8 << 32) | (rsi >> 32)); + r10 = (word64)((r10 << 32) | (rax >> 32)); + /* - a[0] << 32 << 192 */ + rdx = (word64)(rdx - r9); + /* + a[0]-a[2] << 32 << 64 */ + cf = _addcarry_u64(0, rsi, r9, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, rdx, r8, (unsigned long long*)&rdx); + /* a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu */ + r8 = (word64)(0); + /* a += mu << 256 */ + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rsi, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rcx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + /* a += mu << 192 */ + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rsi, (unsigned long long*)&r13); + r10 = (word64)(rsi); + cf = _addcarry_u64(cf, r14, rcx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + /* mu <<= 32 */ + r8 = (word64)((r8 << 32) | (rdx >> 32)); + rdx = (word64)((rdx << 32) | (rcx >> 32)); + rcx = (word64)((rcx << 32) | (rsi >> 32)); + rsi = (word64)((rsi << 32) | (rax >> 32)); + rax = (word64)(rax << 32); + /* a -= (mu << 32) << 192 */ + cf = _subborrow_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, rsi, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, rcx, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rbx, r8, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a += (mu << 32) << 64 */ + cf = _subborrow_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rsi, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rcx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r8, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + rsi = (word64)(0xffffffff00000001); + /* mask m and sub from result if overflow */ + /* m[0] = -1 & mask = mask */ + /* m[2] = 0 & mask = 0 */ + rax = (word32)((word32)r9); + rsi = (word64)(rsi & r9); + cf = _subborrow_u64(0, r13, r9, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, rax, (unsigned long long*)&r14); + WC_S64(rdi, 0) = (word64)(r13); + cf = _subborrow_u64(cf, r15, 0, (unsigned long long*)&r15); + WC_S64(rdi, 8) = (word64)(r14); + cf = _subborrow_u64(cf, rbx, rsi, (unsigned long long*)&rbx); + WC_S64(rdi, 16) = (word64)(r15); + WC_S64(rdi, 24) = (word64)(rbx); + (void)mp; +} + +/* Square the Montgomery form number mod the modulus (prime). (r = a * a mod m) + * + * @param [out] r Result of squaring. + * @param [in] a Number to square in Montgomery form. + * @param [in] m Modulus (prime). + * @param [in] mp Montgomery multiplier. + */ +WOLFSSL_LOCAL void sp_256_mont_sqr_4(sp_digit* r, const sp_digit* a, + const sp_digit* m, sp_digit mp) +{ + word64 rdi, rsi, rax, rdx, r9, r10, r11, r12, r13, r14, r15, r8, rbx, rcx; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(size_t)m; + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r9 = (word64)(rax); + r10 = (word64)(rdx); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r14 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + /* Double */ + r15 = (word64)(0); + cf = _addcarry_u64(0, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r14, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rax = (word64)(rax); + rdx = (word64)(rdx); + r8 = (word64)(rax); + rbx = (word64)(rdx); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rax = (word64)(rax); + rdx = (word64)(rdx); + cf = _addcarry_u64(0, r9, rbx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rax = (word64)(rax); + rdx = (word64)(rdx); + cf = _addcarry_u64(0, r11, rbx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + rbx = (word64)(rdx); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + rax = (word64)(rax); + rdx = (word64)(rdx); + cf = _addcarry_u64(0, r13, rbx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + /* Start Reduction */ + /* mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192 */ + /* - a[0] << 32 << 192 */ + /* a[0]-a[3] + (a[0] * 2) << 192 */ + rax = (word64)(r8); + rdx = (word64)(r11 + r8 * 2); + rsi = (word64)(r9); + rbx = (word64)(r10); + rcx = (word64)(r10); + /* a[0]-a[2] << 32 */ + r8 = (word64)(r8 << 32); + rcx = (word64)((rcx << 32) | (rsi >> 32)); + r9 = (word64)((r9 << 32) | (rax >> 32)); + /* - a[0] << 32 << 192 */ + rdx = (word64)(rdx - r8); + /* + a[0]-a[2] << 32 << 64 */ + cf = _addcarry_u64(0, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, r9, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rdx, rcx, (unsigned long long*)&rdx); + /* a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu */ + rcx = (word64)(0); + /* a += mu << 256 */ + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rsi, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rbx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, r8, r8, (unsigned long long*)&r8); + /* a += mu << 192 */ + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rsi, (unsigned long long*)&r12); + r9 = (word64)(rsi); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + /* mu <<= 32 */ + rcx = (word64)((rcx << 32) | (rdx >> 32)); + rdx = (word64)((rdx << 32) | (rbx >> 32)); + rbx = (word64)((rbx << 32) | (rsi >> 32)); + rsi = (word64)((rsi << 32) | (rax >> 32)); + rax = (word64)(rax << 32); + /* a -= (mu << 32) << 192 */ + cf = _subborrow_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, rsi, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, rbx, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, rcx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* a += (mu << 32) << 64 */ + cf = _subborrow_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rsi, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rbx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + rsi = (word64)(0xffffffff00000001); + /* mask m and sub from result if overflow */ + /* m[0] = -1 & mask = mask */ + /* m[2] = 0 & mask = 0 */ + rax = (word32)((word32)r8); + rsi = (word64)(rsi & r8); + cf = _subborrow_u64(0, r12, r8, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, rax, (unsigned long long*)&r13); + WC_S64(rdi, 0) = (word64)(r12); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rdi, 8) = (word64)(r13); + cf = _subborrow_u64(cf, r15, rsi, (unsigned long long*)&r15); + WC_S64(rdi, 16) = (word64)(r14); + WC_S64(rdi, 24) = (word64)(r15); + (void)mp; +} + +/* Compare a with b in constant time. + * + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + * + * @return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +WOLFSSL_LOCAL sp_int64 sp_256_cmp_4(const sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rcx = (word64)(0); + rdx = (word64)(-1); + rax = (word64)(-1); + r8 = (word64)(1); + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf1 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf1) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf2 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf2) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf3 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf3) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf4 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf4) != (0) ? rcx : rdx; + rax = (word64)(rax ^ rdx); + return (sp_int64)(word64)rax; +} + +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_256_cond_sub_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r12, r13, r14, r15, r8, r9, r10, r11, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r12 = (word64)(WC_L64(rdx, 0)); + r13 = (word64)(WC_L64(rdx, 8)); + r14 = (word64)(WC_L64(rdx, 16)); + r15 = (word64)(WC_L64(rdx, 24)); + r12 = (word64)(r12 & rcx); + r13 = (word64)(r13 & rcx); + r14 = (word64)(r14 & rcx); + r15 = (word64)(r15 & rcx); + r8 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)(WC_L64(rsi, 16)); + r11 = (word64)(WC_L64(rsi, 24)); + cf = _subborrow_u64(0, r8, r12, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, r15, (unsigned long long*)&r11); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Reduce the number back to 256 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_256_mont_reduce_4(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, r8, r9, r10, r11, r12, r13, r14, r15, rax, rdx, rbx, rcx, rsi; + unsigned char cf; + + rdi = (word64)(size_t)a; + + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rdi, 16)); + r11 = (word64)(WC_L64(rdi, 24)); + r12 = (word64)(WC_L64(rdi, 32)); + r13 = (word64)(WC_L64(rdi, 40)); + r14 = (word64)(WC_L64(rdi, 48)); + r15 = (word64)(WC_L64(rdi, 56)); + /* Start Reduction */ + /* mu = a[0]-a[3] + a[0]-a[2] << 32 << 64 + (a[0] * 2) << 192 */ + /* - a[0] << 32 << 192 */ + /* a[0]-a[3] + (a[0] * 2) << 192 */ + rax = (word64)(r8); + rdx = (word64)(r11 + r8 * 2); + rbx = (word64)(r9); + rcx = (word64)(r10); + rsi = (word64)(r10); + /* a[0]-a[2] << 32 */ + r8 = (word64)(r8 << 32); + rsi = (word64)((rsi << 32) | (rbx >> 32)); + r9 = (word64)((r9 << 32) | (rax >> 32)); + /* - a[0] << 32 << 192 */ + rdx = (word64)(rdx - r8); + /* + a[0]-a[2] << 32 << 64 */ + cf = _addcarry_u64(0, rbx, r8, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rcx, r9, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, rdx, rsi, (unsigned long long*)&rdx); + /* a += (mu << 256) - (mu << 224) + (mu << 192) + (mu << 96) - mu */ + rsi = (word64)(0); + /* a += mu << 256 */ + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rbx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rcx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, rdx, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, r8, r8, (unsigned long long*)&r8); + /* a += mu << 192 */ + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rbx, (unsigned long long*)&r12); + r9 = (word64)(rbx); + cf = _addcarry_u64(cf, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + /* mu <<= 32 */ + rsi = (word64)((rsi << 32) | (rdx >> 32)); + rdx = (word64)((rdx << 32) | (rcx >> 32)); + rcx = (word64)((rcx << 32) | (rbx >> 32)); + rbx = (word64)((rbx << 32) | (rax >> 32)); + rax = (word64)(rax << 32); + /* a -= (mu << 32) << 192 */ + cf = _subborrow_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, rbx, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, rcx, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, rdx, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, rsi, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* a += (mu << 32) << 64 */ + cf = _subborrow_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rcx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rsi, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + rbx = (word64)(0xffffffff00000001); + /* mask m and sub from result if overflow */ + /* m[0] = -1 & mask = mask */ + /* m[2] = 0 & mask = 0 */ + rax = (word32)((word32)r8); + rbx = (word64)(rbx & r8); + cf = _subborrow_u64(0, r12, r8, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, rax, (unsigned long long*)&r13); + WC_S64(rdi, 0) = (word64)(r12); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rdi, 8) = (word64)(r13); + cf = _subborrow_u64(cf, r15, rbx, (unsigned long long*)&r15); + WC_S64(rdi, 16) = (word64)(r14); + WC_S64(rdi, 24) = (word64)(r15); + (void)m; + (void)mp; +} + +/* Reduce the number back to 256 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_256_mont_reduce_order_4(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rsi, rcx, r14, r8, r13, r12 = 0, rax = 0, r10 = 0, rdx = 0, + r15 = 0, r9 = 0, r11 = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)mp; + + /* i = 0 */ + r14 = (word64)(0); + r8 = (word64)(4); + r13 = (word64)(rdi); +L_mont_loop_4: + /* mu = a[i] * mp */ + r12 = (word64)(WC_L64(r13, 0)); + r12 = (word64)(r12 * rcx); + /* a[i+0] += m[0] * mu */ + rax = (word64)(WC_L64(rsi, 0)); + r10 = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, r12); + r15 = (word64)(WC_L64(r13, 0)); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + r9 = (word64)(rdx); + WC_S64(r13, 0) = (word64)(r15); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+1] += m[1] * mu */ + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, r12); + r10 = (word64)(WC_L64(rsi, 16)); + r15 = (word64)(WC_L64(r13, 8)); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + r11 = (word64)(rdx); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + WC_S64(r13, 8) = (word64)(r15); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* a[i+2] += m[2] * mu */ + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, r12); + r10 = (word64)(WC_L64(rsi, 24)); + r15 = (word64)(WC_L64(r13, 16)); + cf = _addcarry_u64(0, rax, r11, (unsigned long long*)&rax); + r9 = (word64)(rdx); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + WC_S64(r13, 16) = (word64)(r15); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+3] += m[3] * mu */ + rax = (word64)(r10); + WC_X64I_MUL128(rax, rdx, rax, r12); + r15 = (word64)(WC_L64(r13, 24)); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, r14, (unsigned long long*)&rdx); + r14 = (word64)(0); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(0, r15, rax, (unsigned long long*)&r15); + WC_S64(r13, 24) = (word64)(r15); + cf = _addcarry_u64(cf, WC_L64(r13, 32), rdx, (unsigned long long*)&WC_S64( + r13, 32)); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + /* i += 1 */ + r13 = (word64)(r13 + 8); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_mont_loop_4; + } + rax = (word64)(0); + rdx = (word64)(WC_L64(rdi, 32)); + r8 = (word64)(WC_L64(rdi, 40)); + r15 = (word64)(WC_L64(rdi, 48)); + r9 = (word64)(WC_L64(rdi, 56)); + rax = (word64)(rax - r14); + r10 = (word64)(WC_L64(rsi, 0)); + r11 = (word64)(WC_L64(rsi, 8)); + r12 = (word64)(WC_L64(rsi, 16)); + r13 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(r10 & rax); + r11 = (word64)(r11 & rax); + r12 = (word64)(r12 & rax); + r13 = (word64)(r13 & rax); + cf = _subborrow_u64(0, rdx, r10, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r15, r12, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r15); + WC_S64(rdi, 24) = (word64)(r9); +} + +/* Add two Montgomery form numbers (r = a + b % m). + * + * @param [out] r Result of addition. + * @param [in] a First number to add in Montgomery form. + * @param [in] b Second number to add in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_256_mont_add_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r11, r10; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 8), (unsigned long long*)&rcx); + r11 = (word64)(0xffffffff00000001); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 16), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 24), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, rsi, rsi, (unsigned long long*)&rsi); + r10 = (word32)((word32)rsi); + r11 = (word64)(r11 & rsi); + cf = _subborrow_u64(0, rax, rsi, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r10, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + r10 = (word64)(r10 & rsi); + r11 = (word64)(r11 & rsi); + cf = _subborrow_u64(0, rax, rsi, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r10, (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r9, r11, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + (void)m; +} + +/* Double a Montgomery form number (r = a + a % m). + * + * @param [out] r Result of doubling. + * @param [in] a Number to double in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_256_mont_dbl_4(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r10, r11, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rdx, rdx, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, rax, (unsigned long long*)&rax); + r10 = (word64)(0xffffffff00000001); + cf = _addcarry_u64(cf, rcx, rcx, (unsigned long long*)&rcx); + r11 = (word64)(r8); + cf = _addcarry_u64(cf, r8, r8, (unsigned long long*)&r8); + r11 = (word64)((word64)((sword64)r11 >> 63)); + r9 = (word32)((word32)r11); + r10 = (word64)(r10 & r11); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 8) = (word64)(rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + (void)m; +} + +/* Triple a Montgomery form number (r = a + a + a % m). + * + * @param [out] r Result of Tripling. + * @param [in] a Number to triple in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_256_mont_tpl_4(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r10, r11 = 0, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rdx, rdx, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, rax, (unsigned long long*)&rax); + r10 = (word64)(0xffffffff00000001); + cf = _addcarry_u64(cf, rcx, rcx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r8, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r11, r11, (unsigned long long*)&r11); + r9 = (word32)((word32)r11); + r10 = (word64)(r10 & r11); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 8), (unsigned long long*)&rax); + r10 = (word64)(0xffffffff00000001); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 16), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 24), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r11, r11, (unsigned long long*)&r11); + r9 = (word32)((word32)r11); + r10 = (word64)(r10 & r11); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 8) = (word64)(rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + (void)m; +} + +/* Subtract two Montgomery form numbers (r = a - b % m). + * + * @param [out] r Result of subtration. + * @param [in] a Number to subtract from in Montgomery form. + * @param [in] b Number to subtract with in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_256_mont_sub_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r11, r10; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + cf = _subborrow_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 8), (unsigned long long*)&rcx); + r11 = (word64)(0xffffffff00000001); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 16), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 24), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, rsi, rsi, (unsigned long long*)&rsi); + r10 = (word32)((word32)rsi); + r11 = (word64)(r11 & rsi); + cf = _addcarry_u64(0, rax, rsi, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + r10 = (word64)(r10 & rsi); + r11 = (word64)(r11 & rsi); + cf = _addcarry_u64(0, rax, rsi, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r10, (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + (void)m; +} + +/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_256_mont_div2_4(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r10, r11, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(0xffffffff00000001); + r11 = (word64)(rdx); + r11 = (word64)(r11 & 1); + r11 = (word64)(0 - r11); + r9 = (word32)((word32)r11); + r10 = (word64)(r10 & r11); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + r11 = (word64)(0); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + rdx = (word64)((rdx >> 1) | (rax << 63)); + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r11 << 63)); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + (void)m; +} + +/* Two Montgomery numbers, subtract double second from first (r = a - 2.b % m). + * + * @param [out] r Result of subtration. + * @param [in] a Number to subtract from in Montgomery form. + * @param [in] b Number to double and subtract with in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_256_mont_rsb_sub_dbl_4(sp_digit* r, const sp_digit* a, + sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12, r13, r15, r14; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rdx, 0)); + r11 = (word64)(WC_L64(rdx, 8)); + r12 = (word64)(WC_L64(rdx, 16)); + r13 = (word64)(WC_L64(rdx, 24)); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + r15 = (word64)(0xffffffff00000001); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, rsi, rsi, (unsigned long long*)&rsi); + r14 = (word32)((word32)rsi); + r15 = (word64)(r15 & rsi); + cf = _subborrow_u64(0, r10, rsi, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r15, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + r14 = (word64)(r14 & rsi); + r15 = (word64)(r15 & rsi); + cf = _subborrow_u64(0, r10, rsi, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r15, (unsigned long long*)&r13); + cf = _subborrow_u64(0, rax, r10, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r11, (unsigned long long*)&rcx); + r15 = (word64)(0xffffffff00000001); + cf = _subborrow_u64(cf, r8, r12, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, rsi, rsi, (unsigned long long*)&rsi); + r14 = (word32)((word32)rsi); + r15 = (word64)(r15 & rsi); + cf = _addcarry_u64(0, rax, rsi, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r14, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r15, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + r14 = (word64)(r14 & rsi); + r15 = (word64)(r15 & rsi); + cf = _addcarry_u64(0, rax, rsi, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r14, (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r9, r15, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + r10 = (word64)(WC_L64(rdx, 0)); + r11 = (word64)(WC_L64(rdx, 8)); + r12 = (word64)(WC_L64(rdx, 16)); + r13 = (word64)(WC_L64(rdx, 24)); + cf = _subborrow_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, rcx, (unsigned long long*)&r11); + r15 = (word64)(0xffffffff00000001); + cf = _subborrow_u64(cf, r12, r8, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r9, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, rsi, rsi, (unsigned long long*)&rsi); + r14 = (word32)((word32)rsi); + r15 = (word64)(r15 & rsi); + cf = _addcarry_u64(0, r10, rsi, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r15, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + r14 = (word64)(r14 & rsi); + r15 = (word64)(r15 & rsi); + cf = _addcarry_u64(0, r10, rsi, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r14, (unsigned long long*)&r11); + WC_S64(rdx, 0) = (word64)(r10); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rdx, 8) = (word64)(r11); + cf = _addcarry_u64(cf, r13, r15, (unsigned long long*)&r13); + WC_S64(rdx, 16) = (word64)(r12); + WC_S64(rdx, 24) = (word64)(r13); + (void)m; +} + +#ifndef WC_NO_CACHE_RESISTANT +/* Touch each possible point that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of point to retrieve. + */ +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL void sp_256_get_point_33_4(sp_point_256* r, + const sp_point_256* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0, x1, x2, x3, x4, x5, x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), x13, x14, + x15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi + 0xc8); + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x20); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + x4 = _mm_setzero_si128(); + x5 = _mm_setzero_si128(); + x14 = x15; +L_256_get_point_33_4_start_1: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + rsi = (word64)(rsi + 0xc8); + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + x8 = _mm_and_si128(x8, x12); + x9 = _mm_and_si128(x9, x12); + x10 = _mm_and_si128(x10, x12); + x11 = _mm_and_si128(x11, x12); + x0 = _mm_or_si128(x0, x6); + x1 = _mm_or_si128(x1, x7); + x2 = _mm_or_si128(x2, x8); + x3 = _mm_or_si128(x3, x9); + x4 = _mm_or_si128(x4, x10); + x5 = _mm_or_si128(x5, x11); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_256_get_point_33_4_start_1; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 64), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), x3); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 128), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 144), x5); +} + +#ifdef HAVE_INTEL_AVX2 +/* Touch each possible point that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of point to retrieve. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_256_get_point_33_avx2_4(sp_point_256* r, + const sp_point_256* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m256i y0, y1, y2, y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7, y8, y9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y7 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + rsi = (word64)(rsi + 0xc8); + y9 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + rax = (word64)(0x20); + y8 = _mm256_setzero_si256(); + y7 = _mm256_permutevar8x32_epi32(y7, y8); + y9 = _mm256_permutevar8x32_epi32(y9, y8); + y0 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y8 = y9; +L_256_get_point_33_avx2_4_start: + y6 = _mm256_cmpeq_epi32(y8, y7); + y8 = _mm256_add_epi32(y8, y9); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + rsi = (word64)(rsi + 0xc8); + y3 = _mm256_and_si256(y3, y6); + y4 = _mm256_and_si256(y4, y6); + y5 = _mm256_and_si256(y5, y6); + y0 = _mm256_or_si256(y0, y3); + y1 = _mm256_or_si256(y1, y4); + y2 = _mm256_or_si256(y2, y5); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_256_get_point_33_avx2_4_start; + } + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y2); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WC_NO_CACHE_RESISTANT */ +#ifdef HAVE_INTEL_AVX2 +/* Multiply two Montgomery form numbers mod the modulus (prime). + * (r = a * b mod m) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply in Montgomery form. + * @param [in] b Second number to multiply in Montgomery form. + * @param [in] m Modulus (prime). + * @param [in] mp Montgomery multiplier. + */ +WOLFSSL_LOCAL void sp_256_mont_mul_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m, sp_digit mp) +{ + sp_256_mont_mul_4(r, a, b, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Square the Montgomery form number mod the modulus (prime). (r = a * a mod m) + * + * @param [out] r Result of squaring. + * @param [in] a Number to square in Montgomery form. + * @param [in] m Modulus (prime). + * @param [in] mp Montgomery multiplier. + */ +WOLFSSL_LOCAL void sp_256_mont_sqr_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* m, sp_digit mp) +{ + sp_256_mont_sqr_4(r, a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_256_cond_sub_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r12, r13, r14, r15, r8, r9, r10, r11, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r12 = (word64)(WC_L64(rdx, 0)); + r13 = (word64)(WC_L64(rdx, 8)); + r14 = (word64)(WC_L64(rdx, 16)); + r15 = (word64)(WC_L64(rdx, 24)); + r12 = (word64)(r12 & rcx); + r13 = (word64)(r13 & rcx); + r14 = (word64)(r14 & rcx); + r15 = (word64)(r15 & rcx); + r8 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)(WC_L64(rsi, 16)); + r11 = (word64)(WC_L64(rsi, 24)); + cf = _subborrow_u64(0, r8, r12, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, r15, (unsigned long long*)&r11); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 256 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_256_mont_reduce_order_avx2_4(sp_digit* a, + const sp_digit* m, sp_digit mp) +{ + sp_256_mont_reduce_order_4(a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_256_mont_div2_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r10, r11, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(0xffffffff00000001); + r11 = (word64)(rdx); + r11 = (word64)(r11 & 1); + r11 = (word64)(0 - r11); + r9 = (word32)((word32)r11); + r10 = (word64)(r10 & r11); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + r11 = (word64)(0); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + rdx = (word64)((rdx >> 1) | (rax << 63)); + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r11 << 63)); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + (void)m; +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifndef WC_NO_CACHE_RESISTANT +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL void sp_256_get_entry_64_4(sp_point_256* r, + const sp_table_entry_256* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0, x1, x2, x3, x4 = _mm_setzero_si128(), + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), x9, x10, x11; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + /* From entry 1 */ + rax = (word64)(1); + x9 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi + 0x40); + x11 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x3f); + x11 = _mm_shuffle_epi32(x11, 0); + x9 = _mm_shuffle_epi32(x9, 0); + x10 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + x10 = x11; +L_256_get_entry_64_4_start_0: + x8 = x10; + x10 = _mm_add_epi32(x10, x11); + x8 = _mm_cmpeq_epi32(x8, x9); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + rsi = (word64)(rsi + 0x40); + x4 = _mm_and_si128(x4, x8); + x5 = _mm_and_si128(x5, x8); + x6 = _mm_and_si128(x6, x8); + x7 = _mm_and_si128(x7, x8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_256_get_entry_64_4_start_0; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 64), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), x3); +} + +#ifdef HAVE_INTEL_AVX2 +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_256_get_entry_64_avx2_4(sp_point_256* r, + const sp_table_entry_256* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m256i y0, y1, y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5, y6, y7; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y5 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + rsi = (word64)(rsi + 0x40); + y7 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + rax = (word64)(0x40); + y6 = _mm256_setzero_si256(); + y5 = _mm256_permutevar8x32_epi32(y5, y6); + y7 = _mm256_permutevar8x32_epi32(y7, y6); + y0 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y6 = y7; +L_256_get_entry_64_avx2_4_start: + y4 = _mm256_cmpeq_epi32(y6, y5); + y6 = _mm256_add_epi32(y6, y7); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + rsi = (word64)(rsi + 0x40); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_256_get_entry_64_avx2_4_start; + } + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WC_NO_CACHE_RESISTANT */ +#ifndef WC_NO_CACHE_RESISTANT +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL void sp_256_get_entry_65_4(sp_point_256* r, + const sp_table_entry_256* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0, x1, x2, x3, x4 = _mm_setzero_si128(), + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), x9, x10, x11; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + /* From entry 1 */ + rax = (word64)(1); + x9 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi + 0x40); + x11 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x40); + x11 = _mm_shuffle_epi32(x11, 0); + x9 = _mm_shuffle_epi32(x9, 0); + x10 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + x10 = x11; +L_256_get_entry_65_4_start_0: + x8 = x10; + x10 = _mm_add_epi32(x10, x11); + x8 = _mm_cmpeq_epi32(x8, x9); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + rsi = (word64)(rsi + 0x40); + x4 = _mm_and_si128(x4, x8); + x5 = _mm_and_si128(x5, x8); + x6 = _mm_and_si128(x6, x8); + x7 = _mm_and_si128(x7, x8); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_256_get_entry_65_4_start_0; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 64), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), x3); +} + +#ifdef HAVE_INTEL_AVX2 +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_256_get_entry_65_avx2_4(sp_point_256* r, + const sp_table_entry_256* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m256i y0, y1, y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5, y6, y7; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y5 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + rsi = (word64)(rsi + 0x40); + y7 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + rax = (word64)(0x41); + y6 = _mm256_setzero_si256(); + y5 = _mm256_permutevar8x32_epi32(y5, y6); + y7 = _mm256_permutevar8x32_epi32(y7, y6); + y0 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y6 = y7; +L_256_get_entry_65_avx2_4_start: + y4 = _mm256_cmpeq_epi32(y6, y5); + y6 = _mm256_add_epi32(y6, y7); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + rsi = (word64)(rsi + 0x40); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_256_get_entry_65_avx2_4_start; + } + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WC_NO_CACHE_RESISTANT */ +/* Add 1 to a. (a = a + 1) + * + * @param [in, out] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_256_add_one_4(sp_digit* a) +{ + word64 rdi; + unsigned char cf; + + rdi = (word64)(size_t)a; + + cf = _addcarry_u64(0, WC_L64(rdi, 0), 1, (unsigned long long*)&WC_S64(rdi, + 0)); + cf = _addcarry_u64(cf, WC_L64(rdi, 8), 0, (unsigned long long*)&WC_S64(rdi, + 8)); + cf = _addcarry_u64(cf, WC_L64(rdi, 16), 0, (unsigned long long*)&WC_S64(rdi, + 16)); + cf = _addcarry_u64(cf, WC_L64(rdi, 24), 0, (unsigned long long*)&WC_S64(rdi, + 24)); +} + +/* Read big endian unsigned byte array into r. + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WOLFSSL_LOCAL void sp_256_from_bin_bswap(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, r11, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x20); + r11 = (word64)(0); + goto L_256_from_bin_bswap_64_end; +L_256_from_bin_bswap_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_L64(r9, 56)); + r8 = (word64)(WC_L64(r9, 48)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_L64(r9, 40)); + r8 = (word64)(WC_L64(r9, 32)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(r9, 24)); + r8 = (word64)(WC_L64(r9, 16)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_L64(r9, 8)); + r8 = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_256_from_bin_bswap_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_256_from_bin_bswap_64_start; + } + goto L_256_from_bin_bswap_8_end; +L_256_from_bin_bswap_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_256_from_bin_bswap_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_256_from_bin_bswap_8_start; + } + if ((rcx) == (r11)) { + goto L_256_from_bin_bswap_hi_end; + } + r8 = (word64)(r11); + rax = (word64)(r11); +L_256_from_bin_bswap_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_256_from_bin_bswap_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_256_from_bin_bswap_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_256_from_bin_bswap_zero_end; + } +L_256_from_bin_bswap_zero_start: + WC_S64(rdi, 0) = (word64)(r11); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_256_from_bin_bswap_zero_start; + } +L_256_from_bin_bswap_zero_end: + ; + (void)rsi; +} + +#ifndef NO_MOVBE_SUPPORT +/* Read big endian unsigned byte array into r. + * Uses the movbe instruction which is an optional instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_256_from_bin_movbe(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x20); + goto L_256_from_bin_movbe_64_end; +L_256_from_bin_movbe_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 56))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 48))); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 40))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 32))); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 24))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 16))); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 8))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_256_from_bin_movbe_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_256_from_bin_movbe_64_start; + } + goto L_256_from_bin_movbe_8_end; +L_256_from_bin_movbe_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_256_from_bin_movbe_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_256_from_bin_movbe_8_start; + } + if ((rcx) == (0)) { + goto L_256_from_bin_movbe_hi_end; + } + r8 = (word64)(0); + rax = (word64)(0); +L_256_from_bin_movbe_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_256_from_bin_movbe_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_256_from_bin_movbe_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_256_from_bin_movbe_zero_end; + } +L_256_from_bin_movbe_zero_start: + WC_S64(rdi, 0) = (word64)(0); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_256_from_bin_movbe_zero_start; + } +L_256_from_bin_movbe_zero_end: + ; + (void)rsi; +} + +#endif /* !NO_MOVBE_SUPPORT */ +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 32 + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WOLFSSL_LOCAL void sp_256_to_bin_bswap_4(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rdi, 24)); + rax = (word64)(WC_L64(rdi, 16)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 8)); + rax = (word64)(WC_L64(rdi, 0)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); +} + +#ifndef NO_MOVBE_SUPPORT +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 32 + * Uses the movbe instruction which is optional. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_256_to_bin_movbe_4(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 24))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 16))); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 8))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 0))); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); +} + +#endif /* NO_MOVBE_SUPPORT */ +/* Sub b from a into a. (a -= b) + * + * @param [in, out] a A single precision integer and result. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_256_sub_in_place_4(sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + cf = _subborrow_u64(0, WC_L64(rdi, 0), rdx, (unsigned long long*)&WC_S64( + rdi, 0)); + cf = _subborrow_u64(cf, WC_L64(rdi, 8), rcx, (unsigned long long*)&WC_S64( + rdi, 8)); + cf = _subborrow_u64(cf, WC_L64(rdi, 16), r8, (unsigned long long*)&WC_S64( + rdi, 16)); + cf = _subborrow_u64(cf, WC_L64(rdi, 24), r9, (unsigned long long*)&WC_S64( + rdi, 24)); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_256_mul_d_4(sp_digit* r, const sp_digit* a, sp_digit b) +{ + word64 rdi, rsi, rcx, rax, r10, rdx = 0, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)b; + + /* A[0] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + WC_S64(rdi, 0) = (word64)(r8); + /* A[1] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); +} + +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_256_mul_d_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit b) +{ + sp_256_mul_d_4(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef _WIN64 +/* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) + * + * @param [in] d1 The high order half of the number to divide. + * @param [in] d0 The low order half of the number to divide. + * @param [in] div The dividend. + * + * @return The result of the division. + */ +WOLFSSL_LOCAL sp_digit div_256_word_asm_4(sp_digit d1, sp_digit d0, + sp_digit div) +{ + word64 rdi, rsi, rcx, rax, rdx; + + rdi = (word64)(word64)d1; + rsi = (word64)(word64)d0; + rcx = (word64)(word64)div; + + rax = (word64)(rsi); + rdx = (word64)(rdi); + WC_X64I_DIV128(rax, rdx, rdx, rax, rcx); + return (sp_digit)(word64)rax; +} + +#endif /* _WIN64 */ +#ifdef HAVE_INTEL_AVX2 +/* Multiply two Montgomery form numbers mod the modulus (prime). + * (r = a * b mod m) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply in Montgomery form. + * @param [in] b Second number to multiply in Montgomery form. + */ +WC_X64I_TARGET("bmi2,adx") +WOLFSSL_LOCAL void sp_256_mont_mul_order_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rbp, rdx, r14, r8 = 0, r9 = 0, rbx, rax = 0, r10 = 0, + r11 = 0, r12 = 0, rcx = 0, r15 = 0, r13 = 0; + unsigned char cf; + unsigned char ofl; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rbp = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rsi, 0)); + r14 = (word64)(WC_L64(rbp, 8)); + /* A[0] * B[0] */ + WC_X64I_MUL128(r8, r9, rdx, WC_L64(rbp, 0)); + rbx = (word64)(0); + /* A[0] * B[1] */ + WC_X64I_MUL128(rax, r10, rdx, r14); + cf = _addcarryx_u64(0, r9, rax, (unsigned long long*)&r9); + /* A[0] * B[2] */ + WC_X64I_MUL128(rax, r11, rdx, WC_L64(rbp, 16)); + cf = _addcarryx_u64(cf, r10, rax, (unsigned long long*)&r10); + /* A[0] * B[3] */ + WC_X64I_MUL128(rax, r12, rdx, WC_L64(rbp, 24)); + cf = _addcarryx_u64(cf, r11, rax, (unsigned long long*)&r11); + rdx = (word64)(WC_L64(rsi, 8)); + cf = _addcarryx_u64(cf, r12, rbx, (unsigned long long*)&r12); + /* A[1] * B[0] */ + WC_X64I_MUL128(rax, rcx, rdx, WC_L64(rbp, 0)); + rbx = (word64)(0); + cf = _addcarryx_u64(0, r9, rax, (unsigned long long*)&r9); + /* A[1] * B[1] */ + WC_X64I_MUL128(rax, r15, rdx, r14); + ofl = _addcarryx_u64(0, r10, rcx, (unsigned long long*)&r10); + cf = _addcarryx_u64(cf, r10, rax, (unsigned long long*)&r10); + /* A[1] * B[2] */ + WC_X64I_MUL128(rax, rcx, rdx, WC_L64(rbp, 16)); + ofl = _addcarryx_u64(ofl, r11, r15, (unsigned long long*)&r11); + cf = _addcarryx_u64(cf, r11, rax, (unsigned long long*)&r11); + /* A[1] * B[3] */ + WC_X64I_MUL128(rax, r13, rdx, WC_L64(rbp, 24)); + ofl = _addcarryx_u64(ofl, r12, rcx, (unsigned long long*)&r12); + cf = _addcarryx_u64(cf, r12, rax, (unsigned long long*)&r12); + ofl = _addcarryx_u64(ofl, r13, rbx, (unsigned long long*)&r13); + rdx = (word64)(WC_L64(rsi, 16)); + cf = _addcarryx_u64(cf, r13, rbx, (unsigned long long*)&r13); + /* A[2] * B[0] */ + WC_X64I_MUL128(rax, rcx, rdx, WC_L64(rbp, 0)); + rbx = (word64)(0); + cf = _addcarryx_u64(0, r10, rax, (unsigned long long*)&r10); + /* A[2] * B[1] */ + WC_X64I_MUL128(rax, r15, rdx, r14); + ofl = _addcarryx_u64(0, r11, rcx, (unsigned long long*)&r11); + cf = _addcarryx_u64(cf, r11, rax, (unsigned long long*)&r11); + /* A[2] * B[2] */ + WC_X64I_MUL128(rax, rcx, rdx, WC_L64(rbp, 16)); + ofl = _addcarryx_u64(ofl, r12, r15, (unsigned long long*)&r12); + cf = _addcarryx_u64(cf, r12, rax, (unsigned long long*)&r12); + /* A[2] * B[3] */ + WC_X64I_MUL128(rax, r14, rdx, WC_L64(rbp, 24)); + ofl = _addcarryx_u64(ofl, r13, rcx, (unsigned long long*)&r13); + cf = _addcarryx_u64(cf, r13, rax, (unsigned long long*)&r13); + ofl = _addcarryx_u64(ofl, r14, rbx, (unsigned long long*)&r14); + rdx = (word64)(WC_L64(rsi, 24)); + cf = _addcarryx_u64(cf, r14, rbx, (unsigned long long*)&r14); + /* A[3] * B[0] */ + WC_X64I_MUL128(rax, rcx, rdx, WC_L64(rbp, 0)); + rbx = (word64)(0); + cf = _addcarryx_u64(0, r11, rax, (unsigned long long*)&r11); + /* A[3] * B[1] */ + WC_X64I_MUL128(rax, r15, rdx, WC_L64(rbp, 8)); + ofl = _addcarryx_u64(0, r12, rcx, (unsigned long long*)&r12); + cf = _addcarryx_u64(cf, r12, rax, (unsigned long long*)&r12); + /* A[3] * B[2] */ + WC_X64I_MUL128(rax, rcx, rdx, WC_L64(rbp, 16)); + ofl = _addcarryx_u64(ofl, r13, r15, (unsigned long long*)&r13); + cf = _addcarryx_u64(cf, r13, rax, (unsigned long long*)&r13); + /* A[3] * B[3] */ + WC_X64I_MUL128(rax, r15, rdx, WC_L64(rbp, 24)); + ofl = _addcarryx_u64(ofl, r14, rcx, (unsigned long long*)&r14); + cf = _addcarryx_u64(cf, r14, rax, (unsigned long long*)&r14); + ofl = _addcarryx_u64(ofl, r15, rbx, (unsigned long long*)&r15); + cf = _addcarryx_u64(cf, r15, rbx, (unsigned long long*)&r15); + /* Start Reduction */ + rbx = (word64)(0xccd1c8aaee00bc4f); + /* A[0] */ + rdx = (word64)(rbx); + rdx = (word64)(rdx * r8); + rax = (word64)(0xf3b9cac2fc632551); + rbp = (word64)(0); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xbce6faada7179e84); + cf = _addcarryx_u64(0, r8, rcx, (unsigned long long*)&r8); + ofl = _addcarryx_u64(0, r9, rsi, (unsigned long long*)&r9); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffffffffffff); + cf = _addcarryx_u64(cf, r9, rcx, (unsigned long long*)&r9); + ofl = _addcarryx_u64(ofl, r10, rsi, (unsigned long long*)&r10); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffff00000000); + cf = _addcarryx_u64(cf, r10, rcx, (unsigned long long*)&r10); + ofl = _addcarryx_u64(ofl, r11, rsi, (unsigned long long*)&r11); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + cf = _addcarryx_u64(cf, r11, rcx, (unsigned long long*)&r11); + ofl = _addcarryx_u64(ofl, r12, rsi, (unsigned long long*)&r12); + cf = _addcarryx_u64(cf, r12, rbp, (unsigned long long*)&r12); + r8 = (word64)(rbp); + /* carry */ + ofl = _addcarryx_u64(ofl, r8, rbp, (unsigned long long*)&r8); + cf = _addcarryx_u64(cf, r8, rbp, (unsigned long long*)&r8); + /* A[1] */ + rdx = (word64)(rbx); + rdx = (word64)(rdx * r9); + rax = (word64)(0xf3b9cac2fc632551); + rbp = (word64)(0); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xbce6faada7179e84); + cf = _addcarryx_u64(0, r9, rcx, (unsigned long long*)&r9); + ofl = _addcarryx_u64(0, r10, rsi, (unsigned long long*)&r10); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffffffffffff); + cf = _addcarryx_u64(cf, r10, rcx, (unsigned long long*)&r10); + ofl = _addcarryx_u64(ofl, r11, rsi, (unsigned long long*)&r11); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffff00000000); + cf = _addcarryx_u64(cf, r11, rcx, (unsigned long long*)&r11); + ofl = _addcarryx_u64(ofl, r12, rsi, (unsigned long long*)&r12); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + cf = _addcarryx_u64(cf, r12, rcx, (unsigned long long*)&r12); + ofl = _addcarryx_u64(ofl, r13, rsi, (unsigned long long*)&r13); + cf = _addcarryx_u64(cf, r13, r8, (unsigned long long*)&r13); + r8 = (word64)(rbp); + /* carry */ + ofl = _addcarryx_u64(ofl, r8, rbp, (unsigned long long*)&r8); + cf = _addcarryx_u64(cf, r8, rbp, (unsigned long long*)&r8); + /* A[2] */ + rdx = (word64)(rbx); + rdx = (word64)(rdx * r10); + rax = (word64)(0xf3b9cac2fc632551); + rbp = (word64)(0); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xbce6faada7179e84); + cf = _addcarryx_u64(0, r10, rcx, (unsigned long long*)&r10); + ofl = _addcarryx_u64(0, r11, rsi, (unsigned long long*)&r11); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffffffffffff); + cf = _addcarryx_u64(cf, r11, rcx, (unsigned long long*)&r11); + ofl = _addcarryx_u64(ofl, r12, rsi, (unsigned long long*)&r12); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffff00000000); + cf = _addcarryx_u64(cf, r12, rcx, (unsigned long long*)&r12); + ofl = _addcarryx_u64(ofl, r13, rsi, (unsigned long long*)&r13); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + cf = _addcarryx_u64(cf, r13, rcx, (unsigned long long*)&r13); + ofl = _addcarryx_u64(ofl, r14, rsi, (unsigned long long*)&r14); + cf = _addcarryx_u64(cf, r14, r8, (unsigned long long*)&r14); + r8 = (word64)(rbp); + /* carry */ + ofl = _addcarryx_u64(ofl, r8, rbp, (unsigned long long*)&r8); + cf = _addcarryx_u64(cf, r8, rbp, (unsigned long long*)&r8); + /* A[3] */ + rdx = (word64)(rbx); + rdx = (word64)(rdx * r11); + rax = (word64)(0xf3b9cac2fc632551); + rbp = (word64)(0); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xbce6faada7179e84); + cf = _addcarryx_u64(0, r11, rcx, (unsigned long long*)&r11); + ofl = _addcarryx_u64(0, r12, rsi, (unsigned long long*)&r12); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffffffffffff); + cf = _addcarryx_u64(cf, r12, rcx, (unsigned long long*)&r12); + ofl = _addcarryx_u64(ofl, r13, rsi, (unsigned long long*)&r13); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffff00000000); + cf = _addcarryx_u64(cf, r13, rcx, (unsigned long long*)&r13); + ofl = _addcarryx_u64(ofl, r14, rsi, (unsigned long long*)&r14); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + cf = _addcarryx_u64(cf, r14, rcx, (unsigned long long*)&r14); + ofl = _addcarryx_u64(ofl, r15, rsi, (unsigned long long*)&r15); + cf = _addcarryx_u64(cf, r15, r8, (unsigned long long*)&r15); + r8 = (word64)(rbp); + /* carry */ + ofl = _addcarryx_u64(ofl, r8, rbp, (unsigned long long*)&r8); + cf = _addcarryx_u64(cf, r8, rbp, (unsigned long long*)&r8); + r8 = (word64)(0 - r8); + rax = (word64)(0xf3b9cac2fc632551); + rbx = (word64)(0xbce6faada7179e84); + rax = (word64)(rax & r8); + rbp = (word64)(0xffffffff00000000); + rbx = (word64)(rbx & r8); + rbp = (word64)(rbp & r8); + cf = _subborrow_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, rbx, (unsigned long long*)&r13); + WC_S64(rdi, 0) = (word64)(r12); + cf = _subborrow_u64(cf, r14, r8, (unsigned long long*)&r14); + WC_S64(rdi, 8) = (word64)(r13); + cf = _subborrow_u64(cf, r15, rbp, (unsigned long long*)&r15); + WC_S64(rdi, 16) = (word64)(r14); + WC_S64(rdi, 24) = (word64)(r15); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Square the Montgomery form number mod the modulus (prime). (r = a * a mod m) + * + * @param [out] r Result of squaring. + * @param [in] a Number to square in Montgomery form. + */ +WC_X64I_TARGET("bmi2,adx") +WOLFSSL_LOCAL void sp_256_mont_sqr_order_avx2_4(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, r8, rdx, rcx, rbx, r15, r9 = 0, r10 = 0, r11 = 0, + r12 = 0, rax = 0, r13 = 0, r14 = 0, rbp; + unsigned char ofl; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + r8 = (word64)(0); + rdx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + rbx = (word64)(WC_L64(rsi, 16)); + r15 = (word64)(WC_L64(rsi, 24)); + /* A[0] * A[1] */ + WC_X64I_MUL128(r9, r10, rdx, rcx); + /* A[0] * A[2] */ + WC_X64I_MUL128(r8, r11, rdx, rbx); + ofl = _addcarryx_u64(0, r10, r8, (unsigned long long*)&r10); + /* A[0] * A[3] */ + WC_X64I_MUL128(r8, r12, rdx, r15); + rdx = (word64)(rcx); + ofl = _addcarryx_u64(ofl, r11, r8, (unsigned long long*)&r11); + /* A[1] * A[2] */ + WC_X64I_MUL128(r8, rax, rdx, rbx); + rdx = (word64)(r15); + cf = _addcarryx_u64(0, r11, r8, (unsigned long long*)&r11); + /* A[1] * A[3] */ + WC_X64I_MUL128(r8, r13, rdx, rcx); + r15 = (word64)(0); + ofl = _addcarryx_u64(ofl, r12, rax, (unsigned long long*)&r12); + cf = _addcarryx_u64(cf, r12, r8, (unsigned long long*)&r12); + /* A[2] * A[3] */ + WC_X64I_MUL128(r8, r14, rdx, rbx); + ofl = _addcarryx_u64(ofl, r13, r15, (unsigned long long*)&r13); + cf = _addcarryx_u64(cf, r13, r8, (unsigned long long*)&r13); + ofl = _addcarryx_u64(ofl, r14, r15, (unsigned long long*)&r14); + cf = _addcarryx_u64(cf, r14, r15, (unsigned long long*)&r14); + /* Double with Carry Flag */ + r15 = (word64)(0); + /* A[0] * A[0] */ + rdx = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(r8, rax, rdx, rdx); + cf = _addcarryx_u64(0, r9, r9, (unsigned long long*)&r9); + cf = _addcarryx_u64(cf, r10, r10, (unsigned long long*)&r10); + ofl = _addcarryx_u64(0, r9, rax, (unsigned long long*)&r9); + /* A[1] * A[1] */ + rdx = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rcx, rbx, rdx, rdx); + cf = _addcarryx_u64(cf, r11, r11, (unsigned long long*)&r11); + ofl = _addcarryx_u64(ofl, r10, rcx, (unsigned long long*)&r10); + /* A[2] * A[2] */ + rdx = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rcx, rdx, rdx); + cf = _addcarryx_u64(cf, r12, r12, (unsigned long long*)&r12); + ofl = _addcarryx_u64(ofl, r11, rbx, (unsigned long long*)&r11); + cf = _addcarryx_u64(cf, r13, r13, (unsigned long long*)&r13); + ofl = _addcarryx_u64(ofl, r12, rax, (unsigned long long*)&r12); + cf = _addcarryx_u64(cf, r14, r14, (unsigned long long*)&r14); + /* A[3] * A[3] */ + rdx = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rbx, rdx, rdx); + ofl = _addcarryx_u64(ofl, r13, rcx, (unsigned long long*)&r13); + cf = _addcarryx_u64(cf, r15, r15, (unsigned long long*)&r15); + ofl = _addcarryx_u64(ofl, r14, rax, (unsigned long long*)&r14); + ofl = _addcarryx_u64(ofl, r15, rbx, (unsigned long long*)&r15); + /* Start Reduction */ + rbx = (word64)(0xccd1c8aaee00bc4f); + /* A[0] */ + rdx = (word64)(rbx); + rdx = (word64)(rdx * r8); + rax = (word64)(0xf3b9cac2fc632551); + rbp = (word64)(0); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xbce6faada7179e84); + cf = _addcarryx_u64(0, r8, rcx, (unsigned long long*)&r8); + ofl = _addcarryx_u64(0, r9, rsi, (unsigned long long*)&r9); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffffffffffff); + cf = _addcarryx_u64(cf, r9, rcx, (unsigned long long*)&r9); + ofl = _addcarryx_u64(ofl, r10, rsi, (unsigned long long*)&r10); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffff00000000); + cf = _addcarryx_u64(cf, r10, rcx, (unsigned long long*)&r10); + ofl = _addcarryx_u64(ofl, r11, rsi, (unsigned long long*)&r11); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + cf = _addcarryx_u64(cf, r11, rcx, (unsigned long long*)&r11); + ofl = _addcarryx_u64(ofl, r12, rsi, (unsigned long long*)&r12); + cf = _addcarryx_u64(cf, r12, rbp, (unsigned long long*)&r12); + r8 = (word64)(rbp); + /* carry */ + ofl = _addcarryx_u64(ofl, r8, rbp, (unsigned long long*)&r8); + cf = _addcarryx_u64(cf, r8, rbp, (unsigned long long*)&r8); + /* A[1] */ + rdx = (word64)(rbx); + rdx = (word64)(rdx * r9); + rax = (word64)(0xf3b9cac2fc632551); + rbp = (word64)(0); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xbce6faada7179e84); + cf = _addcarryx_u64(0, r9, rcx, (unsigned long long*)&r9); + ofl = _addcarryx_u64(0, r10, rsi, (unsigned long long*)&r10); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffffffffffff); + cf = _addcarryx_u64(cf, r10, rcx, (unsigned long long*)&r10); + ofl = _addcarryx_u64(ofl, r11, rsi, (unsigned long long*)&r11); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffff00000000); + cf = _addcarryx_u64(cf, r11, rcx, (unsigned long long*)&r11); + ofl = _addcarryx_u64(ofl, r12, rsi, (unsigned long long*)&r12); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + cf = _addcarryx_u64(cf, r12, rcx, (unsigned long long*)&r12); + ofl = _addcarryx_u64(ofl, r13, rsi, (unsigned long long*)&r13); + cf = _addcarryx_u64(cf, r13, r8, (unsigned long long*)&r13); + r8 = (word64)(rbp); + /* carry */ + ofl = _addcarryx_u64(ofl, r8, rbp, (unsigned long long*)&r8); + cf = _addcarryx_u64(cf, r8, rbp, (unsigned long long*)&r8); + /* A[2] */ + rdx = (word64)(rbx); + rdx = (word64)(rdx * r10); + rax = (word64)(0xf3b9cac2fc632551); + rbp = (word64)(0); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xbce6faada7179e84); + cf = _addcarryx_u64(0, r10, rcx, (unsigned long long*)&r10); + ofl = _addcarryx_u64(0, r11, rsi, (unsigned long long*)&r11); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffffffffffff); + cf = _addcarryx_u64(cf, r11, rcx, (unsigned long long*)&r11); + ofl = _addcarryx_u64(ofl, r12, rsi, (unsigned long long*)&r12); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffff00000000); + cf = _addcarryx_u64(cf, r12, rcx, (unsigned long long*)&r12); + ofl = _addcarryx_u64(ofl, r13, rsi, (unsigned long long*)&r13); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + cf = _addcarryx_u64(cf, r13, rcx, (unsigned long long*)&r13); + ofl = _addcarryx_u64(ofl, r14, rsi, (unsigned long long*)&r14); + cf = _addcarryx_u64(cf, r14, r8, (unsigned long long*)&r14); + r8 = (word64)(rbp); + /* carry */ + ofl = _addcarryx_u64(ofl, r8, rbp, (unsigned long long*)&r8); + cf = _addcarryx_u64(cf, r8, rbp, (unsigned long long*)&r8); + /* A[3] */ + rdx = (word64)(rbx); + rdx = (word64)(rdx * r11); + rax = (word64)(0xf3b9cac2fc632551); + rbp = (word64)(0); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xbce6faada7179e84); + cf = _addcarryx_u64(0, r11, rcx, (unsigned long long*)&r11); + ofl = _addcarryx_u64(0, r12, rsi, (unsigned long long*)&r12); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffffffffffff); + cf = _addcarryx_u64(cf, r12, rcx, (unsigned long long*)&r12); + ofl = _addcarryx_u64(ofl, r13, rsi, (unsigned long long*)&r13); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + rax = (word64)(0xffffffff00000000); + cf = _addcarryx_u64(cf, r13, rcx, (unsigned long long*)&r13); + ofl = _addcarryx_u64(ofl, r14, rsi, (unsigned long long*)&r14); + WC_X64I_MUL128(rcx, rsi, rdx, rax); + cf = _addcarryx_u64(cf, r14, rcx, (unsigned long long*)&r14); + ofl = _addcarryx_u64(ofl, r15, rsi, (unsigned long long*)&r15); + cf = _addcarryx_u64(cf, r15, r8, (unsigned long long*)&r15); + r8 = (word64)(rbp); + /* carry */ + ofl = _addcarryx_u64(ofl, r8, rbp, (unsigned long long*)&r8); + cf = _addcarryx_u64(cf, r8, rbp, (unsigned long long*)&r8); + r8 = (word64)(0 - r8); + rax = (word64)(0xf3b9cac2fc632551); + rbx = (word64)(0xbce6faada7179e84); + rax = (word64)(rax & r8); + rbp = (word64)(0xffffffff00000000); + rbx = (word64)(rbx & r8); + rbp = (word64)(rbp & r8); + cf = _subborrow_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, rbx, (unsigned long long*)&r13); + WC_S64(rdi, 0) = (word64)(r12); + cf = _subborrow_u64(cf, r14, r8, (unsigned long long*)&r14); + WC_S64(rdi, 8) = (word64)(r13); + cf = _subborrow_u64(cf, r15, rbp, (unsigned long long*)&r15); + WC_S64(rdi, 16) = (word64)(r14); + WC_S64(rdi, 24) = (word64)(r15); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Non-constant time modular inversion. + * + * @param [out] r Resulting number. + * @param [in] a Number to invert. + * @param [in] m Modulus. + * + * @return MP_OKAY on success. + */ +WOLFSSL_LOCAL void sp_256_mod_inv_4(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rsp, rcx, r8, r9, r10, r11, r12, r13, r14, r15, + rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[68]; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + byte zf7; + byte zf8; + byte zf9; + byte zf10; + byte zf11; + byte zf12; + byte zf13; + byte zf14; + byte zf15; + byte zf16; + byte zf17; + byte zf18; + byte zf19; + byte zf20; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rsp = (word64)(size_t)stk + 544; + rsp = (word64)(rsp - 513); + rcx = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rdx, 24)); + r11 = (word64)(WC_L64(rsi, 0)); + r12 = (word64)(WC_L64(rsi, 8)); + r13 = (word64)(WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rsi, 24)); + r15 = (word64)(0); + if ((((byte)r11 & 1)) != (0)) { + goto L_256_mod_inv_4_v_even_end; + } +L_256_mod_inv_4_v_even_start: + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)((r13 >> 1) | (r14 << 63)); + r14 = (word64)(r14 >> 1); + WC_S8(rsp, r15) = (byte)(1); + r15 = (word64)(r15 + 1); + if ((((byte)r11 & 1)) == (0)) { + goto L_256_mod_inv_4_v_even_start; + } +L_256_mod_inv_4_v_even_end: +L_256_mod_inv_4_uv_start: + zf1 = r10; + zf2 = r14; + if ((r10) < (r14)) { + goto L_256_mod_inv_4_uv_v; + } + if ((zf1) > (zf2)) { + goto L_256_mod_inv_4_uv_u; + } + zf3 = r9; + zf4 = r13; + if ((r9) < (r13)) { + goto L_256_mod_inv_4_uv_v; + } + if ((zf3) > (zf4)) { + goto L_256_mod_inv_4_uv_u; + } + zf5 = r8; + zf6 = r12; + if ((r8) < (r12)) { + goto L_256_mod_inv_4_uv_v; + } + if ((zf5) > (zf6)) { + goto L_256_mod_inv_4_uv_u; + } + if ((rcx) < (r11)) { + goto L_256_mod_inv_4_uv_v; + } +L_256_mod_inv_4_uv_u: + WC_S8(rsp, r15) = (byte)(2); + r15 = (word64)(r15 + 1); + cf = _subborrow_u64(0, rcx, r11, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r12, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)(r10 >> 1); + if ((((byte)rcx & 1)) != (0)) { + goto L_256_mod_inv_4_usubv_even_end; + } +L_256_mod_inv_4_usubv_even_start: + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)(r10 >> 1); + WC_S8(rsp, r15) = (byte)(0); + r15 = (word64)(r15 + 1); + if ((((byte)rcx & 1)) == (0)) { + goto L_256_mod_inv_4_usubv_even_start; + } +L_256_mod_inv_4_usubv_even_end: + if ((rcx) != (1)) { + goto L_256_mod_inv_4_uv_start; + } + rsi = (word64)(r8); + rsi = (word64)(rsi | r9); + if ((rsi) != (0)) { + goto L_256_mod_inv_4_uv_start; + } + rsi = (word64)(rsi | r10); + if ((rsi) != (0)) { + goto L_256_mod_inv_4_uv_start; + } + rax = (rax & ~(word64)0xff) | ((word64)(byte)(1) & 0xff); + goto L_256_mod_inv_4_uv_end; +L_256_mod_inv_4_uv_v: + WC_S8(rsp, r15) = (byte)(3); + r15 = (word64)(r15 + 1); + cf = _subborrow_u64(0, r11, rcx, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, r8, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r9, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, r10, (unsigned long long*)&r14); + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)((r13 >> 1) | (r14 << 63)); + r14 = (word64)(r14 >> 1); + if ((((byte)r11 & 1)) != (0)) { + goto L_256_mod_inv_4_vsubu_even_end; + } +L_256_mod_inv_4_vsubu_even_start: + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)((r13 >> 1) | (r14 << 63)); + r14 = (word64)(r14 >> 1); + WC_S8(rsp, r15) = (byte)(1); + r15 = (word64)(r15 + 1); + if ((((byte)r11 & 1)) == (0)) { + goto L_256_mod_inv_4_vsubu_even_start; + } +L_256_mod_inv_4_vsubu_even_end: + if ((r11) != (1)) { + goto L_256_mod_inv_4_uv_start; + } + rsi = (word64)(r12); + rsi = (word64)(rsi | r13); + if ((rsi) != (0)) { + goto L_256_mod_inv_4_uv_start; + } + rsi = (word64)(rsi | r14); + if ((rsi) != (0)) { + goto L_256_mod_inv_4_uv_start; + } + rax = (rax & ~(word64)0xff) | ((word64)(byte)(0) & 0xff); +L_256_mod_inv_4_uv_end: + rcx = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rdx, 24)); + r11 = (word64)(1); + r12 = (word64)(0); + r13 = (word64)(0); + r14 = (word64)(0); + WC_S8(rsp, r15) = (byte)(7); + rsi = (rsi & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, 0)) & 0xff); + r15 = (word64)(1); + zf7 = (byte)rsi; + zf8 = 1; + if (((byte)rsi) == (1)) { + goto L_256_mod_inv_4_op_div2_d; + } + if ((sword8)(zf7) < (sword8)(zf8)) { + goto L_256_mod_inv_4_op_div2_b; + } + zf9 = (byte)rsi; + zf10 = 3; + if (((byte)rsi) == (3)) { + goto L_256_mod_inv_4_op_d_sub_b; + } + if ((sword8)(zf9) < (sword8)(zf10)) { + goto L_256_mod_inv_4_op_b_sub_d; + } + goto L_256_mod_inv_4_op_end; +L_256_mod_inv_4_op_b_sub_d: + cf = _subborrow_u64(0, rcx, r11, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r12, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + if ((cf) == 0) { + goto L_256_mod_inv_4_op_div2_b; + } + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); +L_256_mod_inv_4_op_div2_b: + zf11 = ((byte)rcx & 1); + rsi = (word64)(0); + if ((zf11) == (0)) { + goto L_256_mod_inv_4_op_div2_b_mod; + } + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); +L_256_mod_inv_4_op_div2_b_mod: + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)((r10 >> 1) | (rsi << 63)); + rsi = (rsi & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, r15)) & 0xff); + r15 = (word64)(r15 + 1); + zf12 = (byte)rsi; + zf13 = 1; + if (((byte)rsi) == (1)) { + goto L_256_mod_inv_4_op_div2_d; + } + if ((sword8)(zf12) < (sword8)(zf13)) { + goto L_256_mod_inv_4_op_div2_b; + } + zf14 = (byte)rsi; + zf15 = 3; + if (((byte)rsi) == (3)) { + goto L_256_mod_inv_4_op_d_sub_b; + } + if ((sword8)(zf14) < (sword8)(zf15)) { + goto L_256_mod_inv_4_op_b_sub_d; + } + goto L_256_mod_inv_4_op_end; +L_256_mod_inv_4_op_d_sub_b: + cf = _subborrow_u64(0, r11, rcx, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, r8, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r9, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, r10, (unsigned long long*)&r14); + if ((cf) == 0) { + goto L_256_mod_inv_4_op_div2_d; + } + cf = _addcarry_u64(0, r11, WC_L64(rdx, 0), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, WC_L64(rdx, 8), (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, WC_L64(rdx, 16), (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, WC_L64(rdx, 24), (unsigned long long*)&r14); +L_256_mod_inv_4_op_div2_d: + zf16 = ((byte)r11 & 1); + rsi = (word64)(0); + if ((zf16) == (0)) { + goto L_256_mod_inv_4_op_div2_d_mod; + } + cf = _addcarry_u64(0, r11, WC_L64(rdx, 0), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, WC_L64(rdx, 8), (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, WC_L64(rdx, 16), (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, WC_L64(rdx, 24), (unsigned long long*)&r14); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); +L_256_mod_inv_4_op_div2_d_mod: + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)((r13 >> 1) | (r14 << 63)); + r14 = (word64)((r14 >> 1) | (rsi << 63)); + rsi = (rsi & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsp, r15)) & 0xff); + r15 = (word64)(r15 + 1); + zf17 = (byte)rsi; + zf18 = 1; + if (((byte)rsi) == (1)) { + goto L_256_mod_inv_4_op_div2_d; + } + if ((sword8)(zf17) < (sword8)(zf18)) { + goto L_256_mod_inv_4_op_div2_b; + } + zf19 = (byte)rsi; + zf20 = 3; + if (((byte)rsi) == (3)) { + goto L_256_mod_inv_4_op_d_sub_b; + } + if ((sword8)(zf19) < (sword8)(zf20)) { + goto L_256_mod_inv_4_op_b_sub_d; + } +L_256_mod_inv_4_op_end: + if (((byte)rax) != (1)) { + goto L_256_mod_inv_4_store_d; + } + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + goto L_256_mod_inv_4_store_end; +L_256_mod_inv_4_store_d: + WC_S64(rdi, 0) = (word64)(r11); + WC_S64(rdi, 8) = (word64)(r12); + WC_S64(rdi, 16) = (word64)(r13); + WC_S64(rdi, 24) = (word64)(r14); +L_256_mod_inv_4_store_end: + ; +} + +#ifdef HAVE_INTEL_AVX2 +XALIGNED(16) static const word32 L_sp256_mod_inv_avx2_4_order[] + WC_X64I_UNUSED = { + 0x00632551, 0x01e84f3b, 0x03bce6fa, 0x03ffffff, + 0x03ff0000, 0x00000000, 0x00000000, 0x00000000, + 0x0272b0bf, 0x02b69c5e, 0x03ffffff, 0x000003ff, + 0x003fffff, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(32) static const word64 L_sp256_mod_inv_avx2_4_one[] WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(16) static const word32 L_sp256_mod_inv_avx2_4_all_one[] + WC_X64I_UNUSED = { + 0x00000001, 0x00000001, 0x00000001, 0x00000001, + 0x00000001, 0x00000001, 0x00000001, 0x00000001, +}; + +XALIGNED(16) static const word32 L_sp256_mod_inv_avx2_4_mask01111[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000001, 0x00000001, + 0x00000001, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_sp256_mod_inv_avx2_4_down_one_dword[] + WC_X64I_UNUSED = { + 0x00000001, 0x00000002, 0x00000003, 0x00000004, + 0x00000005, 0x00000006, 0x00000007, 0x00000007, +}; + +XALIGNED(16) static const word32 L_sp256_mod_inv_avx2_4_neg[] WC_X64I_UNUSED = { + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x80000000, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_sp256_mod_inv_avx2_4_up_one_dword[] + WC_X64I_UNUSED = { + 0x00000007, 0x00000000, 0x00000001, 0x00000002, + 0x00000003, 0x00000007, 0x00000007, 0x00000007, +}; + +XALIGNED(16) static const word32 L_sp256_mod_inv_avx2_4_mask26[] + WC_X64I_UNUSED = { + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x03ffffff, 0x00000000, 0x00000000, 0x00000000, +}; + +/* Non-constant time modular inversion. + * + * @param [out] r Resulting number. + * @param [in] a Number to invert. + * @param [in] m Modulus. + * + * @return MP_OKAY on success. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_256_mod_inv_avx2_4(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12, r13, rbx, r14 = 0, + r15 = 0; + __m256i y0, y1, y2, y3, y4 = _mm256_setzero_si256(), + y5 = _mm256_setzero_si256(), y6, y7, y8, y9, y10, y11, y12, y13, + y14; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rax = (word64)(WC_L64(rdx, 0)); + rcx = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r10 = (word64)(WC_L64(rsi, 0)); + r11 = (word64)(WC_L64(rsi, 8)); + r12 = (word64)(WC_L64(rsi, 16)); + r13 = (word64)(WC_L64(rsi, 24)); + rbx = (word64)((word64)(size_t)L_sp256_mod_inv_avx2_4_order); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + rbx = (word64)((word64)(size_t)L_sp256_mod_inv_avx2_4_one); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_sp256_mod_inv_avx2_4_mask01111); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_sp256_mod_inv_avx2_4_all_one); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_sp256_mod_inv_avx2_4_down_one_dword); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_sp256_mod_inv_avx2_4_neg); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_sp256_mod_inv_avx2_4_up_one_dword); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + rbx = (word64)((word64)(size_t)L_sp256_mod_inv_avx2_4_mask26); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y0 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y1 = _mm256_zextsi128_si256(_mm_setzero_si128()); + y2 = y8; + y3 = _mm256_zextsi128_si256(_mm_setzero_si128()); + if ((((byte)r10 & 1)) != (0)) { + goto L_256_mod_inv_avx2_4_v_even_end; + } +L_256_mod_inv_avx2_4_v_even_start: + r10 = (word64)((r10 >> 1) | (r11 << 63)); + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)(r13 >> 1); + if ((_mm256_testz_si256(y2, y8)) == (1)) { + goto L_256_mod_inv_avx2_4_v_even_shr1; + } + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); +L_256_mod_inv_avx2_4_v_even_shr1: + y4 = _mm256_and_si256(y2, y9); + y5 = _mm256_and_si256(y3, y10); + y4 = _mm256_permutevar8x32_epi32(y4, y11); + y2 = _mm256_srai_epi32(y2, 1); + y3 = _mm256_srai_epi32(y3, 1); + y5 = _mm256_slli_epi32(y5, 25); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y2 = _mm256_add_epi32(y2, y5); + y3 = _mm256_add_epi32(y3, y4); + if ((((byte)r10 & 1)) == (0)) { + goto L_256_mod_inv_avx2_4_v_even_start; + } +L_256_mod_inv_avx2_4_v_even_end: +L_256_mod_inv_avx2_4_uv_start: + zf1 = r9; + zf2 = r13; + if ((r9) < (r13)) { + goto L_256_mod_inv_avx2_4_uv_v; + } + if ((zf1) > (zf2)) { + goto L_256_mod_inv_avx2_4_uv_u; + } + zf3 = r8; + zf4 = r12; + if ((r8) < (r12)) { + goto L_256_mod_inv_avx2_4_uv_v; + } + if ((zf3) > (zf4)) { + goto L_256_mod_inv_avx2_4_uv_u; + } + zf5 = rcx; + zf6 = r11; + if ((rcx) < (r11)) { + goto L_256_mod_inv_avx2_4_uv_v; + } + if ((zf5) > (zf6)) { + goto L_256_mod_inv_avx2_4_uv_u; + } + if ((rax) < (r10)) { + goto L_256_mod_inv_avx2_4_uv_v; + } +L_256_mod_inv_avx2_4_uv_u: + cf = _subborrow_u64(0, rax, r10, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r11, (unsigned long long*)&rcx); + y0 = _mm256_sub_epi32(y0, y2); + cf = _subborrow_u64(cf, r8, r12, (unsigned long long*)&r8); + y1 = _mm256_sub_epi32(y1, y3); + cf = _subborrow_u64(cf, r9, r13, (unsigned long long*)&r9); + if ((_mm256_testz_si256(y1, y12)) == (1)) { + goto L_256_mod_inv_avx2_4_usubv_done_neg; + } + y0 = _mm256_add_epi32(y0, y6); + y1 = _mm256_add_epi32(y1, y7); +L_256_mod_inv_avx2_4_usubv_done_neg: +L_256_mod_inv_avx2_4_usubv_shr1: + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)(r9 >> 1); + if ((_mm256_testz_si256(y0, y8)) == (1)) { + goto L_256_mod_inv_avx2_4_usubv_sub_shr1; + } + y0 = _mm256_add_epi32(y0, y6); + y1 = _mm256_add_epi32(y1, y7); +L_256_mod_inv_avx2_4_usubv_sub_shr1: + y4 = _mm256_and_si256(y0, y9); + y5 = _mm256_and_si256(y1, y10); + y4 = _mm256_permutevar8x32_epi32(y4, y11); + y0 = _mm256_srai_epi32(y0, 1); + y1 = _mm256_srai_epi32(y1, 1); + y5 = _mm256_slli_epi32(y5, 25); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y0 = _mm256_add_epi32(y0, y5); + y1 = _mm256_add_epi32(y1, y4); + if ((((byte)rax & 1)) == (0)) { + goto L_256_mod_inv_avx2_4_usubv_shr1; + } + if ((rax) != (1)) { + goto L_256_mod_inv_avx2_4_uv_start; + } + rsi = (word64)(rcx); + rsi = (word64)(rsi | r8); + if ((rsi) != (0)) { + goto L_256_mod_inv_avx2_4_uv_start; + } + rsi = (word64)(rsi | r9); + if ((rsi) != (0)) { + goto L_256_mod_inv_avx2_4_uv_start; + } + rax = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 0)); + r8 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 1)); + r10 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2)); + r12 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 3)); + rcx = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 0)); + r9 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 1)); + r11 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 2)); + r13 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 3)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + r14 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 0)); + r15 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y1), 0)); + goto L_256_mod_inv_avx2_4_store_done; +L_256_mod_inv_avx2_4_uv_v: + cf = _subborrow_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, rcx, (unsigned long long*)&r11); + y2 = _mm256_sub_epi32(y2, y0); + cf = _subborrow_u64(cf, r12, r8, (unsigned long long*)&r12); + y3 = _mm256_sub_epi32(y3, y1); + cf = _subborrow_u64(cf, r13, r9, (unsigned long long*)&r13); + if ((_mm256_testz_si256(y3, y12)) == (1)) { + goto L_256_mod_inv_avx2_4_vsubu_done_neg; + } + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); +L_256_mod_inv_avx2_4_vsubu_done_neg: +L_256_mod_inv_avx2_4_vsubu_shr1: + r10 = (word64)((r10 >> 1) | (r11 << 63)); + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)(r13 >> 1); + if ((_mm256_testz_si256(y2, y8)) == (1)) { + goto L_256_mod_inv_avx2_4_vsubu_sub_shr1; + } + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); +L_256_mod_inv_avx2_4_vsubu_sub_shr1: + y4 = _mm256_and_si256(y2, y9); + y5 = _mm256_and_si256(y3, y10); + y4 = _mm256_permutevar8x32_epi32(y4, y11); + y2 = _mm256_srai_epi32(y2, 1); + y3 = _mm256_srai_epi32(y3, 1); + y5 = _mm256_slli_epi32(y5, 25); + y4 = _mm256_zextsi128_si256(_mm_slli_epi32(_mm256_castsi256_si128(y4), 25)); + y2 = _mm256_add_epi32(y2, y5); + y3 = _mm256_add_epi32(y3, y4); + if ((((byte)r10 & 1)) == (0)) { + goto L_256_mod_inv_avx2_4_vsubu_shr1; + } + if ((r10) != (1)) { + goto L_256_mod_inv_avx2_4_uv_start; + } + rsi = (word64)(r11); + rsi = (word64)(rsi | r12); + if ((rsi) != (0)) { + goto L_256_mod_inv_avx2_4_uv_start; + } + rsi = (word64)(rsi | r13); + if ((rsi) != (0)) { + goto L_256_mod_inv_avx2_4_uv_start; + } + rax = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 0)); + r8 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 1)); + r10 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 2)); + r12 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 3)); + rcx = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 0)); + r9 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 1)); + r11 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 2)); + r13 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 3)); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y3 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + r14 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y2), 0)); + r15 = (word32)((word32)_mm_extract_epi32(_mm256_castsi256_si128(y3), 0)); +L_256_mod_inv_avx2_4_store_done: + rsi = (word32)((word32)rax); + rax = (word32)((word32)rax & 0x3ffffff); + rsi = (word32)((word32)((sword32)(word32)rsi >> 26)); + rcx = (word32)((word32)rcx + (word32)rsi); + rsi = (word32)((word32)rcx); + rcx = (word32)((word32)rcx & 0x3ffffff); + rsi = (word32)((word32)((sword32)(word32)rsi >> 26)); + r8 = (word32)((word32)r8 + (word32)rsi); + rsi = (word32)((word32)r8); + r8 = (word32)((word32)r8 & 0x3ffffff); + rsi = (word32)((word32)((sword32)(word32)rsi >> 26)); + r9 = (word32)((word32)r9 + (word32)rsi); + rsi = (word32)((word32)r9); + r9 = (word32)((word32)r9 & 0x3ffffff); + rsi = (word32)((word32)((sword32)(word32)rsi >> 26)); + r10 = (word32)((word32)r10 + (word32)rsi); + rsi = (word32)((word32)r10); + r10 = (word32)((word32)r10 & 0x3ffffff); + rsi = (word32)((word32)((sword32)(word32)rsi >> 26)); + r11 = (word32)((word32)r11 + (word32)rsi); + rsi = (word32)((word32)r11); + r11 = (word32)((word32)r11 & 0x3ffffff); + rsi = (word32)((word32)((sword32)(word32)rsi >> 26)); + r12 = (word32)((word32)r12 + (word32)rsi); + rsi = (word32)((word32)r12); + r12 = (word32)((word32)r12 & 0x3ffffff); + rsi = (word32)((word32)((sword32)(word32)rsi >> 26)); + r13 = (word32)((word32)r13 + (word32)rsi); + rsi = (word32)((word32)r13); + r13 = (word32)((word32)r13 & 0x3ffffff); + rsi = (word32)((word32)((sword32)(word32)rsi >> 26)); + r14 = (word32)((word32)r14 + (word32)rsi); + rsi = (word32)((word32)r14); + r14 = (word32)((word32)r14 & 0x3ffffff); + rsi = (word32)((word32)((sword32)(word32)rsi >> 26)); + r15 = (word32)((word32)r15 + (word32)rsi); + rcx = (word64)((word64)(sword64)(sword32)(word32)rcx); + r9 = (word64)((word64)(sword64)(sword32)(word32)r9); + r11 = (word64)((word64)(sword64)(sword32)(word32)r11); + r13 = (word64)((word64)(sword64)(sword32)(word32)r13); + r15 = (word64)((word64)(sword64)(sword32)(word32)r15); + rcx = (word64)(rcx << 26); + r9 = (word64)(r9 << 26); + r11 = (word64)(r11 << 26); + r13 = (word64)(r13 << 26); + r15 = (word64)(r15 << 26); + rax = (word64)((word64)(sword64)(sword32)(word32)rax); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + r8 = (word64)((word64)(sword64)(sword32)(word32)r8); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)((word64)(sword64)(sword32)(word32)r10); + cf = _addcarry_u64(cf, r10, r11, (unsigned long long*)&r10); + r12 = (word64)((word64)(sword64)(sword32)(word32)r12); + cf = _addcarry_u64(cf, r12, r13, (unsigned long long*)&r12); + r14 = (word64)((word64)(sword64)(sword32)(word32)r14); + cf = _addcarry_u64(cf, r14, r15, (unsigned long long*)&r14); + if ((sword64)(r14) >= (sword64)(0)) { + goto L_256_mod_inv_avx2_4_no_add_order; + } + rcx = (word64)(0x9cac2fc632551); + r9 = (word64)(0xada7179e84f3b); + r11 = (word64)(0xfffffffbce6fa); + r13 = (word64)(0xfffffffff); + r15 = (word64)(0xffffffff0000); + rax = (word64)(rax + rcx); + r8 = (word64)(r8 + r9); + r10 = (word64)(r10 + r11); + r12 = (word64)(r12 + r13); + r14 = (word64)(r14 + r15); + rsi = (word64)(0xfffffffffffff); + rcx = (word64)(rax); + rax = (word64)(rax & rsi); + rcx = (word64)((word64)((sword64)rcx >> 52)); + r8 = (word64)(r8 + rcx); + r9 = (word64)(r8); + r8 = (word64)(r8 & rsi); + r9 = (word64)((word64)((sword64)r9 >> 52)); + r10 = (word64)(r10 + r9); + r11 = (word64)(r10); + r10 = (word64)(r10 & rsi); + r11 = (word64)((word64)((sword64)r11 >> 52)); + r12 = (word64)(r12 + r11); + r13 = (word64)(r12); + r12 = (word64)(r12 & rsi); + r13 = (word64)((word64)((sword64)r13 >> 52)); + r14 = (word64)(r14 + r13); +L_256_mod_inv_avx2_4_no_add_order: + rcx = (word64)(r8); + r9 = (word64)(r10); + r11 = (word64)(r12); + rcx = (word64)(rcx << 52); + r8 = (word64)((word64)((sword64)r8 >> 12)); + r9 = (word64)(r9 << 40); + r10 = (word64)((word64)((sword64)r10 >> 24)); + r11 = (word64)(r11 << 28); + r12 = (word64)((word64)((sword64)r12 >> 36)); + r14 = (word64)(r14 << 16); + cf = _addcarry_u64(0, rax, rcx, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r9, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r10, r11, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r12, r14, (unsigned long long*)&r12); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r12); + (void)y13; + (void)y14; +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WOLFSSL_SP_NO_256 */ +#ifdef WOLFSSL_SP_384 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_384_mul_6(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rcx, rsp, rax, rdx = 0, r10, r9, r8; + XALIGNED(32) WC_X64I_SLOT stk[8]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 48); + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 8) = (word64)(r9); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 16) = (word64)(r10); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 24) = (word64)(r8); + /* A[0] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 32) = (word64)(r9); + /* A[0] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 40) = (word64)(r10); + /* A[1] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 48) = (word64)(r8); + /* A[2] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + /* A[3] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r10); + /* A[4] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 72) = (word64)(r8); + /* A[5] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + WC_S64(rdi, 80) = (word64)(r9); + WC_S64(rdi, 88) = (word64)(r10); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r8 = (word64)(WC_L64(rsp, 16)); + r9 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply. + * @param [in] b Second number to multiply. + */ +WOLFSSL_LOCAL void sp_384_mul_avx2_6(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + sp_384_mul_6(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_384_sqr_6(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rax, rdx = 0, r9, r8, rcx, r12, r10, r11; + XALIGNED(32) WC_X64I_SLOT stk[8]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 48); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 8) = (word64)(r8); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 16) = (word64)(r9); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 24) = (word64)(rcx); + /* A[0] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 32) = (word64)(r8); + /* A[0] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 40) = (word64)(r9); + /* A[1] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(rcx); + /* A[2] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[3] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 56) = (word64)(r8); + /* A[3] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 64) = (word64)(r9); + /* A[4] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 72) = (word64)(rcx); + /* A[5] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r10 = (word64)(WC_L64(rsp, 16)); + r11 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); +} + +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * @param [out] r Result of squaring. + * @param [in] a Number to square in Montgomery form. + */ +WOLFSSL_LOCAL void sp_384_sqr_avx2_6(sp_digit* r, const sp_digit* a) +{ + sp_384_sqr_6(r, a); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_384_add_6(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(0); + rcx = (word64)(WC_L64(rsi, 0)); + r8 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(WC_L64(rsi, 16)); + r10 = (word64)(WC_L64(rsi, 24)); + r11 = (word64)(WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rsi, 40)); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, WC_L64(rdx, 32), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, WC_L64(rdx, 40), (unsigned long long*)&r12); + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + WC_S64(rdi, 32) = (word64)(r11); + WC_S64(rdi, 40) = (word64)(r12); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Sub b from a into r. (r = a - b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_384_sub_6(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(0); + rcx = (word64)(WC_L64(rsi, 0)); + r8 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(WC_L64(rsi, 16)); + r10 = (word64)(WC_L64(rsi, 24)); + r11 = (word64)(WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rsi, 40)); + cf = _subborrow_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, WC_L64(rdx, 32), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, WC_L64(rdx, 40), (unsigned long long*)&r12); + WC_S64(rdi, 0) = (word64)(rcx); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + WC_S64(rdi, 32) = (word64)(r11); + WC_S64(rdi, 40) = (word64)(r12); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Conditionally copy a into r using the mask m. + * m is -1 to copy and 0 when not. + * + * @param [out] r A single precision number to copy over. + * @param [in] a A single precision number to copy. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL void sp_384_cond_copy_6(sp_digit* r, const sp_digit* a, + sp_digit m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(word64)m; + + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)(WC_L64(rdi, 16)); + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rdi, 32)); + r11 = (word64)(WC_L64(rdi, 40)); + rax = (word64)(rax ^ WC_L64(rsi, 0)); + rcx = (word64)(rcx ^ WC_L64(rsi, 8)); + r8 = (word64)(r8 ^ WC_L64(rsi, 16)); + r9 = (word64)(r9 ^ WC_L64(rsi, 24)); + r10 = (word64)(r10 ^ WC_L64(rsi, 32)); + r11 = (word64)(r11 ^ WC_L64(rsi, 40)); + rax = (word64)(rax & rdx); + rcx = (word64)(rcx & rdx); + r8 = (word64)(r8 & rdx); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + r11 = (word64)(r11 & rdx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) ^ rax); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) ^ rcx); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) ^ r8); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) ^ r9); + WC_S64(rdi, 32) = (word64)(WC_L64(rdi, 32) ^ r10); + WC_S64(rdi, 40) = (word64)(WC_L64(rdi, 40) ^ r11); +} + +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_384_cond_sub_6(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, r8, r9, rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[8]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 48); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _subborrow_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + WC_S64(rdi, 40) = (word64)(r9); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Reduce the number back to 384 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_384_mont_reduce_6(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, r11, r12, r13, r14, r15, rsi, r10, rbx, rbp, rdx, rax, rcx, + r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)a; + + r11 = (word64)(WC_L64(rdi, 0)); + r12 = (word64)(WC_L64(rdi, 8)); + r13 = (word64)(WC_L64(rdi, 16)); + r14 = (word64)(WC_L64(rdi, 24)); + r15 = (word64)(WC_L64(rdi, 32)); + rsi = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(0); + /* a[0-7] += m[0-5] * mu[0..1] = m[0-5] * (a[0..1] * mp) */ + rbx = (word64)(WC_L64(rdi, 48)); + rbp = (word64)(WC_L64(rdi, 56)); + rdx = (word64)(r11); + rax = (word64)(r12); + rax = (word64)((rax << 32) | (rdx >> 32)); + rdx = (word64)(rdx << 32); + cf = _addcarry_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, r12, (unsigned long long*)&rax); + rax = (word64)(rax + r11); + rcx = (word64)(rdx); + r8 = (word64)(rax); + r9 = (word64)(rax); + r8 = (word64)((r8 << 32) | (rcx >> 32)); + rcx = (word64)(rcx << 32); + r9 = (word64)(r9 >> 32); + cf = _addcarry_u64(0, r11, rcx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r8, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r9, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, rdx, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, rax, (unsigned long long*)&rbp); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _subborrow_u64(0, r13, r8, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, r9, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r15, rax, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _subborrow_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[2-9] += m[0-5] * mu[0..1] = m[0-5] * (a[2..3] * mp) */ + r11 = (word64)(WC_L64(rdi, 64)); + r12 = (word64)(WC_L64(rdi, 72)); + rdx = (word64)(r13); + rax = (word64)(r14); + rax = (word64)((rax << 32) | (rdx >> 32)); + rdx = (word64)(rdx << 32); + cf = _addcarry_u64(0, rdx, r13, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, r14, (unsigned long long*)&rax); + rax = (word64)(rax + r13); + rcx = (word64)(rdx); + r8 = (word64)(rax); + r9 = (word64)(rax); + r8 = (word64)((r8 << 32) | (rcx >> 32)); + rcx = (word64)(rcx << 32); + r9 = (word64)(r9 >> 32); + cf = _addcarry_u64(0, r11, r10, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + r10 = (word64)(0); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r13, rcx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, r8, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r15, r9, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, 0, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _subborrow_u64(0, r15, r8, (unsigned long long*)&r15); + cf = _subborrow_u64(cf, rsi, r9, (unsigned long long*)&rsi); + cf = _subborrow_u64(cf, rbx, rax, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, 0, (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[4-11] += m[0-5] * mu[0..1] = m[0-5] * (a[4..5] * mp) */ + r13 = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(WC_L64(rdi, 88)); + rdx = (word64)(r15); + rax = (word64)(rsi); + rax = (word64)((rax << 32) | (rdx >> 32)); + rdx = (word64)(rdx << 32); + cf = _addcarry_u64(0, rdx, r15, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, rsi, (unsigned long long*)&rax); + rax = (word64)(rax + r15); + rcx = (word64)(rdx); + r8 = (word64)(rax); + r9 = (word64)(rax); + r8 = (word64)((r8 << 32) | (rcx >> 32)); + rcx = (word64)(rcx << 32); + r9 = (word64)(r9 >> 32); + cf = _addcarry_u64(0, r13, r10, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + r10 = (word64)(0); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r15, rcx, (unsigned long long*)&r15); + cf = _addcarry_u64(cf, rsi, r8, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, rbx, r9, (unsigned long long*)&rbx); + cf = _addcarry_u64(cf, rbp, 0, (unsigned long long*)&rbp); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rax, (unsigned long long*)&r9); + rax = (word64)(0); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _subborrow_u64(0, rbx, r8, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, r9, (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r11, rax, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + /* Subtract mod if carry */ + r10 = (word64)(0 - r10); + r9 = (word64)(0xfffffffffffffffe); + rcx = (word32)((word32)r10); + r8 = (word64)(r10); + r9 = (word64)(r9 & r10); + r8 = (word64)(r8 << 32); + cf = _subborrow_u64(0, rbx, rcx, (unsigned long long*)&rbx); + cf = _subborrow_u64(cf, rbp, r8, (unsigned long long*)&rbp); + cf = _subborrow_u64(cf, r11, r9, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, r10, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, r10, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, r10, (unsigned long long*)&r14); + WC_S64(rdi, 0) = (word64)(rbx); + WC_S64(rdi, 8) = (word64)(rbp); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + WC_S64(rdi, 32) = (word64)(r13); + WC_S64(rdi, 40) = (word64)(r14); + (void)m; + (void)mp; +} + +/* Reduce the number back to 384 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_384_mont_reduce_order_6(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rsi, rcx, r15, r8, r13, r14, r11 = 0, rax = 0, r10 = 0, + rdx = 0, r9 = 0, r12 = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)mp; + + r15 = (word64)(0); + /* i = 6 */ + r8 = (word64)(6); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 8)); +L_384_mont_reduce_order_6_loop: + /* mu = a[i] * mp */ + r11 = (word64)(r13); + r11 = (word64)(r11 * rcx); + /* a[i+0] += m[0] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* a[i+1] += m[1] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r13, r10, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+2] += m[2] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+3] += m[3] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 24) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+4] += m[4] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rdi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+5] += m[5] * mu */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, r15, (unsigned long long*)&rdx); + r15 = (word64)(0); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 40) = (word64)(r12); + cf = _addcarry_u64(cf, WC_L64(rdi, 48), rdx, (unsigned long long*)&WC_S64( + rdi, 48)); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* i -= 1 */ + rdi = (word64)(rdi + 8); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_384_mont_reduce_order_6_loop; + } + WC_S64(rdi, 0) = (word64)(r13); + WC_S64(rdi, 8) = (word64)(r14); + r15 = (word64)(0 - r15); +#ifdef _WIN64 + rdx = (word64)(rsi); + rcx = (word64)(r15); +#else + rcx = (word64)(r15); + rdx = (word64)(rsi); +#endif /* _WIN64 */ + rsi = (word64)(rdi); + rdi = (word64)(rdi); + rdi = (word64)(rdi - 0x30); + (void)sp_384_cond_sub_6((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx, (sp_digit)rcx); +} + +/* Compare a with b in constant time. + * + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + * + * @return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +WOLFSSL_LOCAL sp_int64 sp_384_cmp_6(const sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rcx = (word64)(0); + rdx = (word64)(-1); + rax = (word64)(-1); + r8 = (word64)(1); + r9 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf1 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf1) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf2 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf2) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf3 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf3) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf4 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf4) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf5 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf5) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf6 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf6) != (0) ? rcx : rdx; + rax = (word64)(rax ^ rdx); + return (sp_int64)(word64)rax; +} + +/* Add two Montgomery form numbers (r = a + b % m). + * + * @param [out] r Result of addition. + * @param [in] a First number to add in Montgomery form. + * @param [in] b Second number to add in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_384_mont_add_6(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r13, r14, r12; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rsi, 32)); + r11 = (word64)(WC_L64(rsi, 40)); + cf = _addcarry_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 8), (unsigned long long*)&rcx); + r13 = (word64)(0xffffffff00000000); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 16), (unsigned long long*)&r8); + r14 = (word64)(0xfffffffffffffffe); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 24), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 32), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, WC_L64(rdx, 40), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, rsi, rsi, (unsigned long long*)&rsi); + r12 = (word32)((word32)rsi); + r13 = (word64)(r13 & rsi); + r14 = (word64)(r14 & rsi); + cf = _subborrow_u64(0, rax, r12, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r13, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r14, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, rsi, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, rsi, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, rsi, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + r12 = (word64)(r12 & rsi); + r13 = (word64)(r13 & rsi); + r14 = (word64)(r14 & rsi); + cf = _subborrow_u64(0, rax, r12, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r13, (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + cf = _subborrow_u64(cf, r8, r14, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, r9, rsi, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + cf = _subborrow_u64(cf, r10, rsi, (unsigned long long*)&r10); + WC_S64(rdi, 24) = (word64)(r9); + cf = _subborrow_u64(cf, r11, rsi, (unsigned long long*)&r11); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(r11); + (void)m; +} + +/* Double a Montgomery form number (r = a + a % m). + * + * @param [out] r Result of doubling. + * @param [in] a Number to double in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_384_mont_dbl_6(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r12, r13, r14, r11; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)(WC_L64(rsi, 40)); + cf = _addcarry_u64(0, rdx, rdx, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, rax, (unsigned long long*)&rax); + r12 = (word64)(0xffffffff00000000); + cf = _addcarry_u64(cf, rcx, rcx, (unsigned long long*)&rcx); + r13 = (word64)(0xfffffffffffffffe); + cf = _addcarry_u64(cf, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + r14 = (word64)(r10); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + r14 = (word64)((word64)((sword64)r14 >> 63)); + r11 = (word32)((word32)r14); + r12 = (word64)(r12 & r14); + r13 = (word64)(r13 & r14); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r12, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r13, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r14, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r14, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + r11 = (word64)(r11 & r14); + r12 = (word64)(r12 & r14); + r13 = (word64)(r13 & r14); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r12, (unsigned long long*)&rax); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, r13, (unsigned long long*)&rcx); + WC_S64(rdi, 8) = (word64)(rax); + cf = _subborrow_u64(cf, r8, r14, (unsigned long long*)&r8); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _subborrow_u64(cf, r9, r14, (unsigned long long*)&r9); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); + (void)m; +} + +/* Double a Montgomery form number (r = a + a % m). + * + * @param [out] r Result of doubling. + * @param [in] a Number to double in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_384_mont_tpl_6(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r12, r13, r14 = 0, r11; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)(WC_L64(rsi, 40)); + cf = _addcarry_u64(0, rdx, rdx, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, rax, (unsigned long long*)&rax); + r12 = (word64)(0xffffffff00000000); + cf = _addcarry_u64(cf, rcx, rcx, (unsigned long long*)&rcx); + r13 = (word64)(0xfffffffffffffffe); + cf = _addcarry_u64(cf, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r14, r14, (unsigned long long*)&r14); + r11 = (word32)((word32)r14); + r12 = (word64)(r12 & r14); + r13 = (word64)(r13 & r14); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r12, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r13, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r14, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r14, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + r11 = (word64)(r11 & r14); + r12 = (word64)(r12 & r14); + r13 = (word64)(r13 & r14); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r12, (unsigned long long*)&rax); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, r13, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r14, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r14, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + cf = _addcarry_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 8), (unsigned long long*)&rax); + r12 = (word64)(0xffffffff00000000); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 16), (unsigned long long*)&rcx); + r13 = (word64)(0xfffffffffffffffe); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 24), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 32), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rsi, 40), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r14, r14, (unsigned long long*)&r14); + r11 = (word32)((word32)r14); + r12 = (word64)(r12 & r14); + r13 = (word64)(r13 & r14); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r12, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r13, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, r14, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, r14, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + r11 = (word64)(r11 & r14); + r12 = (word64)(r12 & r14); + r13 = (word64)(r13 & r14); + cf = _subborrow_u64(0, rdx, r11, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, r12, (unsigned long long*)&rax); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, r13, (unsigned long long*)&rcx); + WC_S64(rdi, 8) = (word64)(rax); + cf = _subborrow_u64(cf, r8, r14, (unsigned long long*)&r8); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _subborrow_u64(cf, r9, r14, (unsigned long long*)&r9); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r10, r14, (unsigned long long*)&r10); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); + (void)m; +} + +/* Subtract two Montgomery form numbers (r = a - b % m). + * + * @param [out] r Result of subtration. + * @param [in] a Number to subtract from in Montgomery form. + * @param [in] b Number to subtract with in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_384_mont_sub_6(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r13, r14, r12; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rsi, 32)); + r11 = (word64)(WC_L64(rsi, 40)); + cf = _subborrow_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 8), (unsigned long long*)&rcx); + r13 = (word64)(0xffffffff00000000); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 16), (unsigned long long*)&r8); + r14 = (word64)(0xfffffffffffffffe); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 24), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 32), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, WC_L64(rdx, 40), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, rsi, rsi, (unsigned long long*)&rsi); + r12 = (word32)((word32)rsi); + r13 = (word64)(r13 & rsi); + r14 = (word64)(r14 & rsi); + cf = _addcarry_u64(0, rax, r12, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r13, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r14, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rsi, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rsi, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rsi, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + r12 = (word64)(r12 & rsi); + r13 = (word64)(r13 & rsi); + r14 = (word64)(r14 & rsi); + cf = _addcarry_u64(0, rax, r12, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r13, (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + cf = _addcarry_u64(cf, r8, r14, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _addcarry_u64(cf, r9, rsi, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + cf = _addcarry_u64(cf, r10, rsi, (unsigned long long*)&r10); + WC_S64(rdi, 24) = (word64)(r9); + cf = _addcarry_u64(cf, r11, rsi, (unsigned long long*)&r11); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(r11); + (void)m; +} + +/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_384_mont_div2_6(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rsp, r11, r10, rax, r8, rcx; + XALIGNED(32) WC_X64I_SLOT stk[8]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 48); + r11 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)(0); + rax = (word64)(r11); + r11 = (word64)(r11 & 1); + r11 = (word64)(0 - r11); + r8 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 8) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 16)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 24) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 32)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 40) = (word64)(r8); + cf = _addcarry_u64(0, WC_L64(rsp, 0), rax, (unsigned long long*)&WC_S64(rsp, + 0)); + rax = (word64)(WC_L64(rsi, 8)); + cf = _addcarry_u64(cf, WC_L64(rsp, 8), rax, (unsigned long long*)&WC_S64( + rsp, 8)); + rax = (word64)(WC_L64(rsi, 16)); + cf = _addcarry_u64(cf, WC_L64(rsp, 16), rax, (unsigned long long*)&WC_S64( + rsp, 16)); + rax = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(cf, WC_L64(rsp, 24), rax, (unsigned long long*)&WC_S64( + rsp, 24)); + rax = (word64)(WC_L64(rsi, 32)); + cf = _addcarry_u64(cf, WC_L64(rsp, 32), rax, (unsigned long long*)&WC_S64( + rsp, 32)); + rax = (word64)(WC_L64(rsi, 40)); + cf = _addcarry_u64(cf, WC_L64(rsp, 40), rax, (unsigned long long*)&WC_S64( + rsp, 40)); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + rax = (word64)(WC_L64(rsp, 0)); + rcx = (word64)(WC_L64(rsp, 8)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 0) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 16)); + rcx = (word64)((rcx >> 1) | (rax << 63)); + WC_S64(rdi, 8) = (word64)(rcx); + rcx = (word64)(WC_L64(rsp, 24)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 16) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 32)); + rcx = (word64)((rcx >> 1) | (rax << 63)); + WC_S64(rdi, 24) = (word64)(rcx); + rcx = (word64)(WC_L64(rsp, 40)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 32) = (word64)(rax); + rcx = (word64)((rcx >> 1) | (r10 << 63)); + WC_S64(rdi, 40) = (word64)(rcx); +} + +#ifndef WC_NO_CACHE_RESISTANT +/* Touch each possible point that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of point to retrieve. + */ +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL void sp_384_get_point_33_6(sp_point_384* r, + const sp_point_384* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0, x1, x2, x3, x4, x5, x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), x13, x14, + x15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi + 0x128); + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x20); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + x4 = _mm_setzero_si128(); + x5 = _mm_setzero_si128(); + x14 = x15; +L_384_get_point_33_6_start_1: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 96)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 112)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + rsi = (word64)(rsi + 0x128); + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + x8 = _mm_and_si128(x8, x12); + x9 = _mm_and_si128(x9, x12); + x10 = _mm_and_si128(x10, x12); + x11 = _mm_and_si128(x11, x12); + x0 = _mm_or_si128(x0, x6); + x1 = _mm_or_si128(x1, x7); + x2 = _mm_or_si128(x2, x8); + x3 = _mm_or_si128(x3, x9); + x4 = _mm_or_si128(x4, x10); + x5 = _mm_or_si128(x5, x11); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_384_get_point_33_6_start_1; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 96), x3); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 112), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 128), x5); + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi - 0x2500); + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x20); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x14 = x15; +L_384_get_point_33_6_start_2: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 208)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); + rsi = (word64)(rsi + 0x128); + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + x8 = _mm_and_si128(x8, x12); + x0 = _mm_or_si128(x0, x6); + x1 = _mm_or_si128(x1, x7); + x2 = _mm_or_si128(x2, x8); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_384_get_point_33_6_start_2; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 192), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 208), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 224), x2); +} + +#ifdef HAVE_INTEL_AVX2 +/* Touch each possible point that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of point to retrieve. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_384_get_point_33_avx2_6(sp_point_384* r, + const sp_point_384* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m128i x1, x3, x5, x7 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(); + __m256i y0, y2, y4, y6 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13, y14, y15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y13 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + rsi = (word64)(rsi + 0x128); + y15 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + rax = (word64)(0x20); + y14 = _mm256_setzero_si256(); + y13 = _mm256_permutevar8x32_epi32(y13, y14); + y15 = _mm256_permutevar8x32_epi32(y15, y14); + y0 = _mm256_setzero_si256(); + x1 = _mm_setzero_si128(); + y2 = _mm256_setzero_si256(); + x3 = _mm_setzero_si128(); + y4 = _mm256_setzero_si256(); + x5 = _mm_setzero_si128(); + y14 = y15; +L_384_get_point_33_avx2_6_start: + y12 = _mm256_cmpeq_epi32(y14, y13); + y14 = _mm256_add_epi32(y14, y15); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 128)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 224)); + rsi = (word64)(rsi + 0x128); + y6 = _mm256_and_si256(y6, y12); + x7 = _mm_and_si128(x7, _mm256_castsi256_si128(y12)); + y8 = _mm256_and_si256(y8, y12); + x9 = _mm_and_si128(x9, _mm256_castsi256_si128(y12)); + y10 = _mm256_and_si256(y10, y12); + x11 = _mm_and_si128(x11, _mm256_castsi256_si128(y12)); + y0 = _mm256_or_si256(y0, y6); + x1 = _mm_or_si128(x1, x7); + y2 = _mm256_or_si256(y2, y8); + x3 = _mm_or_si128(x3, x9); + y4 = _mm256_or_si256(y4, y10); + x5 = _mm_or_si128(x5, x11); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_384_get_point_33_avx2_6_start; + } + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), x1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 128), x3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 224), x5); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WC_NO_CACHE_RESISTANT */ +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 384 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_384_mont_reduce_order_avx2_6(sp_digit* a, + const sp_digit* m, sp_digit mp) +{ + sp_384_mont_reduce_order_6(a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_384_cond_sub_avx2_6(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r10, r8, r9, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _subborrow_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + WC_S64(rdi, 40) = (word64)(r10); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus (prime). + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_384_mont_div2_avx2_6(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, r11, r10, r8, rax, rcx, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + r11 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)(0); + r8 = (word64)(r11); + r11 = (word64)(r11 & 1); + r11 = (word64)(0 - r11); + rax = (word64)(WC_L64(rdx, 0)); + rcx = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(WC_L64(rsi, 8)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + rax = (word64)(WC_L64(rdx, 16)); + rcx = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rdx, 32)); + rcx = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(WC_L64(rsi, 40)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + WC_S64(rdi, 40) = (word64)(r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rdi, 16)); + r9 = (word64)((r9 >> 1) | (r8 << 63)); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rdi, 24)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rdi, 32)); + r9 = (word64)((r9 >> 1) | (r8 << 63)); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rdi, 40)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 32) = (word64)(r8); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + WC_S64(rdi, 40) = (word64)(r9); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifndef WC_NO_CACHE_RESISTANT +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL void sp_384_get_entry_64_6(sp_point_384* r, + const sp_table_entry_384* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0, x1, x2, x3, x4, x5, x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), x13, x14, + x15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + /* From entry 1 */ + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi + 0x60); + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x3f); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + x4 = _mm_setzero_si128(); + x5 = _mm_setzero_si128(); + x14 = x15; +L_384_get_entry_64_6_start_0: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + rsi = (word64)(rsi + 0x60); + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + x8 = _mm_and_si128(x8, x12); + x9 = _mm_and_si128(x9, x12); + x10 = _mm_and_si128(x10, x12); + x11 = _mm_and_si128(x11, x12); + x0 = _mm_or_si128(x0, x6); + x1 = _mm_or_si128(x1, x7); + x2 = _mm_or_si128(x2, x8); + x3 = _mm_or_si128(x3, x9); + x4 = _mm_or_si128(x4, x10); + x5 = _mm_or_si128(x5, x11); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_384_get_entry_64_6_start_0; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 96), x3); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 112), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 128), x5); +} + +#ifdef HAVE_INTEL_AVX2 +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_384_get_entry_64_avx2_6(sp_point_384* r, + const sp_table_entry_384* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m128i x1, x3, x5 = _mm_setzero_si128(), x7 = _mm_setzero_si128(); + __m256i y0, y2, y4 = _mm256_setzero_si256(), y6 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9, y10, y11; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y9 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + rsi = (word64)(rsi + 0x60); + y11 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + rax = (word64)(0x40); + y10 = _mm256_setzero_si256(); + y9 = _mm256_permutevar8x32_epi32(y9, y10); + y11 = _mm256_permutevar8x32_epi32(y11, y10); + y0 = _mm256_setzero_si256(); + x1 = _mm_setzero_si128(); + y2 = _mm256_setzero_si256(); + x3 = _mm_setzero_si128(); + y10 = y11; +L_384_get_entry_64_avx2_6_start: + y8 = _mm256_cmpeq_epi32(y10, y9); + y10 = _mm256_add_epi32(y10, y11); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 48)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + rsi = (word64)(rsi + 0x60); + y4 = _mm256_and_si256(y4, y8); + x5 = _mm_and_si128(x5, _mm256_castsi256_si128(y8)); + y6 = _mm256_and_si256(y6, y8); + x7 = _mm_and_si128(x7, _mm256_castsi256_si128(y8)); + y0 = _mm256_or_si256(y0, y4); + x1 = _mm_or_si128(x1, x5); + y2 = _mm256_or_si256(y2, y6); + x3 = _mm_or_si128(x3, x7); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_384_get_entry_64_avx2_6_start; + } + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), x1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 128), x3); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WC_NO_CACHE_RESISTANT */ +#ifndef WC_NO_CACHE_RESISTANT +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL void sp_384_get_entry_65_6(sp_point_384* r, + const sp_table_entry_384* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m128i x0, x1, x2, x3, x4, x5, x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), x13, x14, + x15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + /* From entry 1 */ + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi + 0x60); + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x40); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + x4 = _mm_setzero_si128(); + x5 = _mm_setzero_si128(); + x14 = x15; +L_384_get_entry_65_6_start_0: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 64)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + rsi = (word64)(rsi + 0x60); + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + x8 = _mm_and_si128(x8, x12); + x9 = _mm_and_si128(x9, x12); + x10 = _mm_and_si128(x10, x12); + x11 = _mm_and_si128(x11, x12); + x0 = _mm_or_si128(x0, x6); + x1 = _mm_or_si128(x1, x7); + x2 = _mm_or_si128(x2, x8); + x3 = _mm_or_si128(x3, x9); + x4 = _mm_or_si128(x4, x10); + x5 = _mm_or_si128(x5, x11); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_384_get_entry_65_6_start_0; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 96), x3); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 112), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 128), x5); +} + +#ifdef HAVE_INTEL_AVX2 +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_384_get_entry_65_avx2_6(sp_point_384* r, + const sp_table_entry_384* table, int idx) +{ + word64 rdi, rsi, rdx, rax; + __m128i x1, x3, x5 = _mm_setzero_si128(), x7 = _mm_setzero_si128(); + __m256i y0, y2, y4 = _mm256_setzero_si256(), y6 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9, y10, y11; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + rax = (word64)(1); + y9 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + rsi = (word64)(rsi + 0x60); + y11 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + rax = (word64)(0x41); + y10 = _mm256_setzero_si256(); + y9 = _mm256_permutevar8x32_epi32(y9, y10); + y11 = _mm256_permutevar8x32_epi32(y11, y10); + y0 = _mm256_setzero_si256(); + x1 = _mm_setzero_si128(); + y2 = _mm256_setzero_si256(); + x3 = _mm_setzero_si128(); + y10 = y11; +L_384_get_entry_65_avx2_6_start: + y8 = _mm256_cmpeq_epi32(y10, y9); + y10 = _mm256_add_epi32(y10, y11); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 48)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 80)); + rsi = (word64)(rsi + 0x60); + y4 = _mm256_and_si256(y4, y8); + x5 = _mm_and_si128(x5, _mm256_castsi256_si128(y8)); + y6 = _mm256_and_si256(y6, y8); + x7 = _mm_and_si128(x7, _mm256_castsi256_si128(y8)); + y0 = _mm256_or_si256(y0, y4); + x1 = _mm_or_si128(x1, x5); + y2 = _mm256_or_si256(y2, y6); + x3 = _mm_or_si128(x3, x7); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_384_get_entry_65_avx2_6_start; + } + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), x1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 128), x3); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WC_NO_CACHE_RESISTANT */ +/* Add 1 to a. (a = a + 1) + * + * @param [in, out] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_384_add_one_6(sp_digit* a) +{ + word64 rdi; + unsigned char cf; + + rdi = (word64)(size_t)a; + + cf = _addcarry_u64(0, WC_L64(rdi, 0), 1, (unsigned long long*)&WC_S64(rdi, + 0)); + cf = _addcarry_u64(cf, WC_L64(rdi, 8), 0, (unsigned long long*)&WC_S64(rdi, + 8)); + cf = _addcarry_u64(cf, WC_L64(rdi, 16), 0, (unsigned long long*)&WC_S64(rdi, + 16)); + cf = _addcarry_u64(cf, WC_L64(rdi, 24), 0, (unsigned long long*)&WC_S64(rdi, + 24)); + cf = _addcarry_u64(cf, WC_L64(rdi, 32), 0, (unsigned long long*)&WC_S64(rdi, + 32)); + cf = _addcarry_u64(cf, WC_L64(rdi, 40), 0, (unsigned long long*)&WC_S64(rdi, + 40)); +} + +/* Read big endian unsigned byte array into r. + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WOLFSSL_LOCAL void sp_384_from_bin_bswap(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, r11, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x30); + r11 = (word64)(0); + goto L_384_from_bin_bswap_64_end; +L_384_from_bin_bswap_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_L64(r9, 56)); + r8 = (word64)(WC_L64(r9, 48)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_L64(r9, 40)); + r8 = (word64)(WC_L64(r9, 32)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(r9, 24)); + r8 = (word64)(WC_L64(r9, 16)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_L64(r9, 8)); + r8 = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_384_from_bin_bswap_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_384_from_bin_bswap_64_start; + } + goto L_384_from_bin_bswap_8_end; +L_384_from_bin_bswap_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_384_from_bin_bswap_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_384_from_bin_bswap_8_start; + } + if ((rcx) == (r11)) { + goto L_384_from_bin_bswap_hi_end; + } + r8 = (word64)(r11); + rax = (word64)(r11); +L_384_from_bin_bswap_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_384_from_bin_bswap_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_384_from_bin_bswap_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_384_from_bin_bswap_zero_end; + } +L_384_from_bin_bswap_zero_start: + WC_S64(rdi, 0) = (word64)(r11); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_384_from_bin_bswap_zero_start; + } +L_384_from_bin_bswap_zero_end: + ; + (void)rsi; +} + +#ifndef NO_MOVBE_SUPPORT +/* Read big endian unsigned byte array into r. + * Uses the movbe instruction which is an optional instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_384_from_bin_movbe(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x30); + goto L_384_from_bin_movbe_64_end; +L_384_from_bin_movbe_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 56))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 48))); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 40))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 32))); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 24))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 16))); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 8))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_384_from_bin_movbe_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_384_from_bin_movbe_64_start; + } + goto L_384_from_bin_movbe_8_end; +L_384_from_bin_movbe_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_384_from_bin_movbe_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_384_from_bin_movbe_8_start; + } + if ((rcx) == (0)) { + goto L_384_from_bin_movbe_hi_end; + } + r8 = (word64)(0); + rax = (word64)(0); +L_384_from_bin_movbe_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_384_from_bin_movbe_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_384_from_bin_movbe_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_384_from_bin_movbe_zero_end; + } +L_384_from_bin_movbe_zero_start: + WC_S64(rdi, 0) = (word64)(0); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_384_from_bin_movbe_zero_start; + } +L_384_from_bin_movbe_zero_end: + ; + (void)rsi; +} + +#endif /* !NO_MOVBE_SUPPORT */ +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 48 + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WOLFSSL_LOCAL void sp_384_to_bin_bswap_6(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rdi, 40)); + rax = (word64)(WC_L64(rdi, 32)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 24)); + rax = (word64)(WC_L64(rdi, 16)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 8)); + rax = (word64)(WC_L64(rdi, 0)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 32) = (word64)(rdx); + WC_S64(rsi, 40) = (word64)(rax); +} + +#ifndef NO_MOVBE_SUPPORT +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 48 + * Uses the movbe instruction which is optional. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_384_to_bin_movbe_6(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rdx, rax; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 40))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 32))); + WC_S64(rsi, 0) = (word64)(rdx); + WC_S64(rsi, 8) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 24))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 16))); + WC_S64(rsi, 16) = (word64)(rdx); + WC_S64(rsi, 24) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 8))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 0))); + WC_S64(rsi, 32) = (word64)(rdx); + WC_S64(rsi, 40) = (word64)(rax); +} + +#endif /* NO_MOVBE_SUPPORT */ +/* Sub b from a into a. (a -= b) + * + * @param [in, out] a A single precision integer and result. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_384_sub_in_place_6(sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, r10, r11, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rsi, 32)); + r11 = (word64)(WC_L64(rsi, 40)); + cf = _subborrow_u64(0, WC_L64(rdi, 0), rdx, (unsigned long long*)&WC_S64( + rdi, 0)); + cf = _subborrow_u64(cf, WC_L64(rdi, 8), rcx, (unsigned long long*)&WC_S64( + rdi, 8)); + cf = _subborrow_u64(cf, WC_L64(rdi, 16), r8, (unsigned long long*)&WC_S64( + rdi, 16)); + cf = _subborrow_u64(cf, WC_L64(rdi, 24), r9, (unsigned long long*)&WC_S64( + rdi, 24)); + cf = _subborrow_u64(cf, WC_L64(rdi, 32), r10, (unsigned long long*)&WC_S64( + rdi, 32)); + cf = _subborrow_u64(cf, WC_L64(rdi, 40), r11, (unsigned long long*)&WC_S64( + rdi, 40)); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_384_mul_d_6(sp_digit* r, const sp_digit* a, sp_digit b) +{ + word64 rdi, rsi, rcx, rax, r10, rdx = 0, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)b; + + /* A[0] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + WC_S64(rdi, 0) = (word64)(r8); + /* A[1] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r10); + WC_S64(rdi, 48) = (word64)(r8); +} + +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_384_mul_d_avx2_6(sp_digit* r, const sp_digit* a, + const sp_digit b) +{ + sp_384_mul_d_6(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef _WIN64 +/* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) + * + * @param [in] d1 The high order half of the number to divide. + * @param [in] d0 The low order half of the number to divide. + * @param [in] div The dividend. + * + * @return The result of the division. + */ +WOLFSSL_LOCAL sp_digit div_384_word_asm_6(sp_digit d1, sp_digit d0, + sp_digit div) +{ + word64 rdi, rsi, rcx, rax, rdx; + + rdi = (word64)(word64)d1; + rsi = (word64)(word64)d0; + rcx = (word64)(word64)div; + + rax = (word64)(rsi); + rdx = (word64)(rdi); + WC_X64I_DIV128(rax, rdx, rdx, rax, rcx); + return (sp_digit)(word64)rax; +} + +#endif /* _WIN64 */ +/* Shift number right by 1 bit. (r = a >> 1) + * + * @param [out] r Result of right shift by 1. + * @param [in] a Number to shift. + */ +WOLFSSL_LOCAL void sp_384_rshift1_6(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)((rdx >> 1) | (rax << 63)); + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)(r10 >> 1); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); +} + +/* Divide the number by 2 mod the prime. (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus + */ +WOLFSSL_LOCAL void sp_384_div2_mod_6(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12, r13, r14, r15, rbx, + rbp; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rsi, 32)); + r11 = (word64)(WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rdx, 0)); + r13 = (word64)(WC_L64(rdx, 8)); + r14 = (word64)(WC_L64(rdx, 16)); + r15 = (word64)(WC_L64(rdx, 24)); + rbx = (word64)(WC_L64(rdx, 32)); + rbp = (word64)(WC_L64(rdx, 40)); + rdx = (word64)(rax); + rdx = (word64)(rdx & 1); + if ((rdx) == (0)) { + goto L_384_mod_inv_6_div2_mod_no_add; + } + cf = _addcarry_u64(0, rax, r12, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r13, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r14, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r15, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rbx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rbp, (unsigned long long*)&r11); + rdx = (word64)(0); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); +L_384_mod_inv_6_div2_mod_no_add: + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)((r10 >> 1) | (r11 << 63)); + r11 = (word64)((r11 >> 1) | (rdx << 63)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(r11); +} + +WOLFSSL_LOCAL int sp_384_num_bits_6(const sp_digit * a) +{ + word64 rdi, rax, rdx; + + rdi = (word64)(size_t)a; + + rax = (word64)(0); + rdx = (word64)(WC_L64(rdi, 40)); + if ((rdx) == (0)) { + goto L_384_num_bits_6_end_320; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x141); + goto L_384_num_bits_6_done; +L_384_num_bits_6_end_320: + rdx = (word64)(WC_L64(rdi, 32)); + if ((rdx) == (0)) { + goto L_384_num_bits_6_end_256; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x101); + goto L_384_num_bits_6_done; +L_384_num_bits_6_end_256: + rdx = (word64)(WC_L64(rdi, 24)); + if ((rdx) == (0)) { + goto L_384_num_bits_6_end_192; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0xc1); + goto L_384_num_bits_6_done; +L_384_num_bits_6_end_192: + rdx = (word64)(WC_L64(rdi, 16)); + if ((rdx) == (0)) { + goto L_384_num_bits_6_end_128; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x81); + goto L_384_num_bits_6_done; +L_384_num_bits_6_end_128: + rdx = (word64)(WC_L64(rdi, 8)); + if ((rdx) == (0)) { + goto L_384_num_bits_6_end_64; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x41); + goto L_384_num_bits_6_done; +L_384_num_bits_6_end_64: + rdx = (word64)(WC_L64(rdi, 0)); + if ((rdx) == (0)) { + goto L_384_num_bits_6_end_0; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 1); + goto L_384_num_bits_6_done; +L_384_num_bits_6_end_0: +L_384_num_bits_6_done: + return (int)(word64)rax; +} + +#endif /* WOLFSSL_SP_384 */ +#ifdef WOLFSSL_SP_521 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_521_mul_9(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rcx, rsp, rax, rdx = 0, r10, r9, r8; + XALIGNED(32) WC_X64I_SLOT stk[12]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 96; + rsp = (word64)(rsp - 72); + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 8) = (word64)(r9); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 16) = (word64)(r10); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 24) = (word64)(r8); + /* A[0] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 32) = (word64)(r9); + /* A[0] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 40) = (word64)(r10); + /* A[0] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 48) = (word64)(r8); + /* A[0] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 56) = (word64)(r9); + /* A[0] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 64) = (word64)(r10); + /* A[1] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 72) = (word64)(r8); + /* A[2] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 80) = (word64)(r9); + /* A[3] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 88) = (word64)(r10); + /* A[4] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 96) = (word64)(r8); + /* A[5] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + /* A[6] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r10); + /* A[7] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 120) = (word64)(r8); + /* A[8] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + WC_S64(rdi, 128) = (word64)(r9); + WC_S64(rdi, 136) = (word64)(r10); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r8 = (word64)(WC_L64(rsp, 16)); + r9 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + r8 = (word64)(WC_L64(rsp, 48)); + r9 = (word64)(WC_L64(rsp, 56)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 64)); + WC_S64(rdi, 64) = (word64)(rax); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply. + * @param [in] b Second number to multiply. + */ +WOLFSSL_LOCAL void sp_521_mul_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + sp_521_mul_9(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_521_sqr_9(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rax, rdx = 0, r9, r8, rcx, r12, r10, r11; + XALIGNED(32) WC_X64I_SLOT stk[12]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 96; + rsp = (word64)(rsp - 72); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 8) = (word64)(r8); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 16) = (word64)(r9); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 24) = (word64)(rcx); + /* A[0] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 32) = (word64)(r8); + /* A[0] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 40) = (word64)(r9); + /* A[0] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 48) = (word64)(rcx); + /* A[0] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rsp, 56) = (word64)(r8); + /* A[0] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 64) = (word64)(r9); + /* A[1] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[2] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rdi, 72) = (word64)(rcx); + /* A[2] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[3] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 80) = (word64)(r8); + /* A[3] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[4] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + /* A[4] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(rcx); + /* A[5] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[6] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 104) = (word64)(r8); + /* A[6] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 112) = (word64)(r9); + /* A[7] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 120) = (word64)(rcx); + /* A[8] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 128) = (word64)(r8); + WC_S64(rdi, 136) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r10 = (word64)(WC_L64(rsp, 16)); + r11 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + r10 = (word64)(WC_L64(rsp, 48)); + r11 = (word64)(WC_L64(rsp, 56)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(r10); + WC_S64(rdi, 56) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 64)); + WC_S64(rdi, 64) = (word64)(rax); +} + +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_521_sqr_avx2_9(sp_digit* r, const sp_digit* a) +{ + sp_521_sqr_9(r, a); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_521_add_9(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax, r8; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Add */ + rcx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(0); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Sub b from a into r. (r = a - b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_521_sub_9(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, r8, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rcx = (word64)(WC_L64(rsi, 0)); + cf = _subborrow_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Conditionally copy a into r using the mask m. + * m is -1 to copy and 0 when not. + * + * @param [out] r A single precision number to copy over. + * @param [in] a A single precision number to copy. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL void sp_521_cond_copy_9(sp_digit* r, const sp_digit* a, + sp_digit m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(word64)m; + + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)(WC_L64(rdi, 16)); + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rdi, 32)); + rax = (word64)(rax ^ WC_L64(rsi, 0)); + rcx = (word64)(rcx ^ WC_L64(rsi, 8)); + r8 = (word64)(r8 ^ WC_L64(rsi, 16)); + r9 = (word64)(r9 ^ WC_L64(rsi, 24)); + r10 = (word64)(r10 ^ WC_L64(rsi, 32)); + rax = (word64)(rax & rdx); + rcx = (word64)(rcx & rdx); + r8 = (word64)(r8 & rdx); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) ^ rax); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) ^ rcx); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) ^ r8); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) ^ r9); + WC_S64(rdi, 32) = (word64)(WC_L64(rdi, 32) ^ r10); + rax = (word64)(WC_L64(rdi, 40)); + rcx = (word64)(WC_L64(rdi, 48)); + r8 = (word64)(WC_L64(rdi, 56)); + r9 = (word64)(WC_L64(rdi, 64)); + rax = (word64)(rax ^ WC_L64(rsi, 40)); + rcx = (word64)(rcx ^ WC_L64(rsi, 48)); + r8 = (word64)(r8 ^ WC_L64(rsi, 56)); + r9 = (word64)(r9 ^ WC_L64(rsi, 64)); + rax = (word64)(rax & rdx); + rcx = (word64)(rcx & rdx); + r8 = (word64)(r8 & rdx); + r9 = (word64)(r9 & rdx); + WC_S64(rdi, 40) = (word64)(WC_L64(rdi, 40) ^ rax); + WC_S64(rdi, 48) = (word64)(WC_L64(rdi, 48) ^ rcx); + WC_S64(rdi, 56) = (word64)(WC_L64(rdi, 56) ^ r8); + WC_S64(rdi, 64) = (word64)(WC_L64(rdi, 64) ^ r9); +} + +/* Multiply two Montgomery form numbers mod the modulus (prime). + * (r = a * b mod m) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply in Montgomery form. + * @param [in] b Second number to multiply in Montgomery form. + * @param [in] m Modulus (prime). + * @param [in] mp Montgomery multiplier. + */ +WOLFSSL_LOCAL void sp_521_mont_mul_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m, sp_digit mp) +{ + word64 rdi, rsi, rcx, rsp, rax, rdx = 0, r13, r12, r11, r10, r8, r9; + XALIGNED(32) WC_X64I_SLOT stk[20]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 144); + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r13 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r12 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rsp, 8) = (word64)(r12); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 16) = (word64)(r13); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 24) = (word64)(r11); + /* A[0] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[4] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rsp, 32) = (word64)(r12); + /* A[0] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 40) = (word64)(r13); + /* A[0] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[1] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[4] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[5] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[6] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 48) = (word64)(r11); + /* A[0] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[1] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[2] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[3] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[4] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[5] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[6] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[7] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rsp, 56) = (word64)(r12); + /* A[0] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[1] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 64) = (word64)(r13); + /* A[1] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[4] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[5] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[6] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[7] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[8] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 72) = (word64)(r11); + /* A[2] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[3] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[4] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[5] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[6] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[7] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[8] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rsp, 80) = (word64)(r12); + /* A[3] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 88) = (word64)(r13); + /* A[4] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[5] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[6] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[7] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[8] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 96) = (word64)(r11); + /* A[5] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r11 = (word64)(0); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[6] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[7] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + /* A[8] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + WC_S64(rsp, 104) = (word64)(r12); + /* A[6] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r12 = (word64)(0); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + WC_S64(rsp, 112) = (word64)(r13); + /* A[7] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r13 = (word64)(0); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[8] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rsp, 120) = (word64)(r11); + /* A[8] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, rdx, (unsigned long long*)&r13); + WC_S64(rsp, 128) = (word64)(r12); + WC_S64(rsp, 136) = (word64)(r13); + rax = (word64)(WC_L64(rsp, 64)); + rdx = (word64)(WC_L64(rsp, 72)); + r11 = (word64)(WC_L64(rsp, 80)); + r10 = (word64)(rax); + r10 = (word64)(r10 & 0x1ff); + r12 = (word64)(WC_L64(rsp, 88)); + r13 = (word64)(WC_L64(rsp, 96)); + rcx = (word64)(WC_L64(rsp, 104)); + rsi = (word64)(WC_L64(rsp, 112)); + r8 = (word64)(WC_L64(rsp, 120)); + r9 = (word64)(WC_L64(rsp, 128)); + rax = (word64)((rax >> 9) | (rdx << 55)); + rdx = (word64)((rdx >> 9) | (r11 << 55)); + r11 = (word64)((r11 >> 9) | (r12 << 55)); + r12 = (word64)((r12 >> 9) | (r13 << 55)); + r13 = (word64)((r13 >> 9) | (rcx << 55)); + rcx = (word64)((rcx >> 9) | (rsi << 55)); + rsi = (word64)((rsi >> 9) | (r8 << 55)); + r8 = (word64)((r8 >> 9) | (r9 << 55)); + r9 = (word64)(r9 >> 9); + cf = _addcarry_u64(0, rax, WC_L64(rsp, 0), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, WC_L64(rsp, 8), (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r11, WC_L64(rsp, 16), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, WC_L64(rsp, 24), (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, WC_L64(rsp, 32), (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rcx, WC_L64(rsp, 40), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, rsi, WC_L64(rsp, 48), (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 56), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r10, r9, (unsigned long long*)&r10); + r9 = (word64)(r10); + r10 = (word64)(r10 >> 9); + r9 = (word64)(r9 & 0x1ff); + cf = _addcarry_u64(0, rax, r10, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r11); + WC_S64(rdi, 24) = (word64)(r12); + WC_S64(rdi, 32) = (word64)(r13); + WC_S64(rdi, 40) = (word64)(rcx); + WC_S64(rdi, 48) = (word64)(rsi); + WC_S64(rdi, 56) = (word64)(r8); + WC_S64(rdi, 64) = (word64)(r9); + (void)m; + (void)mp; +} + +/* Square the Montgomery form number mod the modulus (prime). (r = a * a mod m) + * + * @param [out] r Result of squaring. + * @param [in] a Number to square in Montgomery form. + * @param [in] m Modulus (prime). + * @param [in] mp Montgomery multiplier. + */ +WOLFSSL_LOCAL void sp_521_mont_sqr_9(sp_digit* r, const sp_digit* a, + const sp_digit* m, sp_digit mp) +{ + word64 rdi, rsi, rsp, rax, rdx, r10, r9, r8, r13, r11, r12, rcx; + XALIGNED(32) WC_X64I_SLOT stk[20]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(size_t)m; + rsp = (word64)(size_t)stk + 160; + rsp = (word64)(rsp - 144); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r10 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 8) = (word64)(r9); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 16) = (word64)(r10); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 24) = (word64)(r8); + /* A[0] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 32) = (word64)(r9); + /* A[0] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + /* A[1] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r10, r11, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r13, (unsigned long long*)&r9); + WC_S64(rsp, 40) = (word64)(r10); + /* A[0] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + /* A[1] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + WC_S64(rsp, 48) = (word64)(r8); + /* A[0] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + /* A[1] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r12, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, r13, (unsigned long long*)&r8); + WC_S64(rsp, 56) = (word64)(r9); + /* A[0] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + /* A[1] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[2] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[4] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r10, r11, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r13, (unsigned long long*)&r9); + WC_S64(rsp, 64) = (word64)(r10); + /* A[1] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r10 = (word64)(0); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + /* A[2] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[3] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[4] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r13, (unsigned long long*)&r10); + WC_S64(rsp, 72) = (word64)(r8); + /* A[2] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r8 = (word64)(0); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + /* A[3] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[4] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[5] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r12, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, r13, (unsigned long long*)&r8); + WC_S64(rsp, 80) = (word64)(r9); + /* A[3] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r9 = (word64)(0); + r13 = (word64)(0); + r11 = (word64)(rax); + r12 = (word64)(rdx); + /* A[4] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + /* A[5] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r11, rax, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, rdx, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(0, r10, r11, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r13, (unsigned long long*)&r9); + WC_S64(rsp, 88) = (word64)(r10); + /* A[4] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 96) = (word64)(r8); + /* A[5] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 104) = (word64)(r9); + /* A[6] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 112) = (word64)(r10); + /* A[7] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 120) = (word64)(r8); + /* A[8] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + WC_S64(rsp, 128) = (word64)(r9); + WC_S64(rsp, 136) = (word64)(r10); + r8 = (word64)(WC_L64(rsp, 64)); + r9 = (word64)(WC_L64(rsp, 72)); + r10 = (word64)(WC_L64(rsp, 80)); + rcx = (word64)(r8); + rcx = (word64)(rcx & 0x1ff); + rax = (word64)(WC_L64(rsp, 88)); + rdx = (word64)(WC_L64(rsp, 96)); + r11 = (word64)(WC_L64(rsp, 104)); + r12 = (word64)(WC_L64(rsp, 112)); + r13 = (word64)(WC_L64(rsp, 120)); + rsi = (word64)(WC_L64(rsp, 128)); + r8 = (word64)((r8 >> 9) | (r9 << 55)); + r9 = (word64)((r9 >> 9) | (r10 << 55)); + r10 = (word64)((r10 >> 9) | (rax << 55)); + rax = (word64)((rax >> 9) | (rdx << 55)); + rdx = (word64)((rdx >> 9) | (r11 << 55)); + r11 = (word64)((r11 >> 9) | (r12 << 55)); + r12 = (word64)((r12 >> 9) | (r13 << 55)); + r13 = (word64)((r13 >> 9) | (rsi << 55)); + rsi = (word64)(rsi >> 9); + cf = _addcarry_u64(0, r8, WC_L64(rsp, 0), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsp, 8), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rsp, 16), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rax, WC_L64(rsp, 24), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, WC_L64(rsp, 32), (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r11, WC_L64(rsp, 40), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, WC_L64(rsp, 48), (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, WC_L64(rsp, 56), (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rcx, rsi, (unsigned long long*)&rcx); + rsi = (word64)(rcx); + rcx = (word64)(rcx >> 9); + rsi = (word64)(rsi & 0x1ff); + cf = _addcarry_u64(0, r8, rcx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rdx, 0, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, rsi, 0, (unsigned long long*)&rsi); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(rax); + WC_S64(rdi, 32) = (word64)(rdx); + WC_S64(rdi, 40) = (word64)(r11); + WC_S64(rdi, 48) = (word64)(r12); + WC_S64(rdi, 56) = (word64)(r13); + WC_S64(rdi, 64) = (word64)(rsi); + (void)mp; +} + +/* Compare a with b in constant time. + * + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + * + * @return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +WOLFSSL_LOCAL sp_int64 sp_521_cmp_9(const sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + word64 zf7; + word64 zf8; + word64 zf9; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rcx = (word64)(0); + rdx = (word64)(-1); + rax = (word64)(-1); + r8 = (word64)(1); + r9 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf1 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf1) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(WC_L64(rsi, 56)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf2 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf2) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf3 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf3) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf4 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf4) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf5 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf5) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf6 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf6) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf7 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf7) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf8 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf8) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf9 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf9) != (0) ? rcx : rdx; + rax = (word64)(rax ^ rdx); + return (sp_int64)(word64)rax; +} + +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_521_cond_sub_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, r8, r9, rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[12]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 96; + rsp = (word64)(rsp - 72); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r8 = (word64)(r8 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _subborrow_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + WC_S64(rdi, 64) = (word64)(r8); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Reduce the number back to 521 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_521_mont_reduce_9(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rdx, rax, rcx, r14, r8, r9, r10, r11, r12, r13; + unsigned char cf; + + rdi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rdi, 64)); + rax = (word64)(WC_L64(rdi, 72)); + rcx = (word64)(WC_L64(rdi, 80)); + r14 = (word64)(rdx); + r14 = (word64)(r14 & 0x1ff); + r8 = (word64)(WC_L64(rdi, 88)); + r9 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(WC_L64(rdi, 104)); + r11 = (word64)(WC_L64(rdi, 112)); + r12 = (word64)(WC_L64(rdi, 120)); + r13 = (word64)(WC_L64(rdi, 128)); + rdx = (word64)((rdx >> 9) | (rax << 55)); + rax = (word64)((rax >> 9) | (rcx << 55)); + rcx = (word64)((rcx >> 9) | (r8 << 55)); + r8 = (word64)((r8 >> 9) | (r9 << 55)); + r9 = (word64)((r9 >> 9) | (r10 << 55)); + r10 = (word64)((r10 >> 9) | (r11 << 55)); + r11 = (word64)((r11 >> 9) | (r12 << 55)); + r12 = (word64)((r12 >> 9) | (r13 << 55)); + r13 = (word64)(r13 >> 9); + cf = _addcarry_u64(0, rdx, WC_L64(rdi, 0), (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, WC_L64(rdi, 8), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rdi, 16), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdi, 24), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdi, 32), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdi, 40), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, WC_L64(rdi, 48), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, WC_L64(rdi, 56), (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r14, r13, (unsigned long long*)&r14); + r13 = (word64)(r14); + r14 = (word64)(r14 >> 9); + r13 = (word64)(r13 & 0x1ff); + cf = _addcarry_u64(0, rdx, r14, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); + WC_S64(rdi, 48) = (word64)(r11); + WC_S64(rdi, 56) = (word64)(r12); + WC_S64(rdi, 64) = (word64)(r13); + (void)m; + (void)mp; +} + +/* Reduce the number back to 521 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_521_mont_reduce_order_9(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rsi, rcx, r15, r8, r13, r14, r11 = 0, rax = 0, r10 = 0, + rdx = 0, r9 = 0, r12 = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)mp; + + r15 = (word64)(0); + /* i = 9 */ + r8 = (word64)(9); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 8)); +L_521_mont_reduce_order_9_loop: + /* mu = a[i] * mp */ + r11 = (word64)(r13); + r11 = (word64)(r11 * rcx); + if ((r8) != (1)) { + goto L_521_mont_reduce_order_9_nomask; + } + r11 = (word64)(r11 & 0x1ff); +L_521_mont_reduce_order_9_nomask: + /* a[i+0] += m[0] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + WC_S64(rdi, 0) = (word64)(r13); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* a[i+1] += m[1] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r13, r10, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+2] += m[2] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+3] += m[3] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 24) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+4] += m[4] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rdi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+5] += m[5] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 40) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+6] += m[6] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rdi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 48) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+7] += m[7] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r12 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 56) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+8] += m[8] * mu */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r12 = (word64)(WC_L64(rdi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rdx, r15, (unsigned long long*)&rdx); + r15 = (word64)(0); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 64) = (word64)(r12); + cf = _addcarry_u64(cf, WC_L64(rdi, 72), rdx, (unsigned long long*)&WC_S64( + rdi, 72)); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* i -= 1 */ + rdi = (word64)(rdi + 8); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_521_mont_reduce_order_9_loop; + } + WC_S64(rdi, 0) = (word64)(r13); + WC_S64(rdi, 8) = (word64)(r14); + rcx = (word64)(rdi); + rdi = (word64)(rdi - 0x48); + rcx = (word64)(rcx - 8); + rax = (word64)(WC_L64(rcx, 0)); + rdx = (word64)(WC_L64(rcx, 8)); + r8 = (word64)(WC_L64(rcx, 16)); + r9 = (word64)(WC_L64(rcx, 24)); + r11 = (word64)(WC_L64(rcx, 32)); + rax = (word64)((rax >> 9) | (rdx << 55)); + rdx = (word64)((rdx >> 9) | (r8 << 55)); + r8 = (word64)((r8 >> 9) | (r9 << 55)); + r9 = (word64)((r9 >> 9) | (r11 << 55)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rdx = (word64)(WC_L64(rcx, 40)); + r8 = (word64)(WC_L64(rcx, 48)); + r9 = (word64)(WC_L64(rcx, 56)); + rax = (word64)(WC_L64(rcx, 64)); + r11 = (word64)((r11 >> 9) | (rdx << 55)); + rdx = (word64)((rdx >> 9) | (r8 << 55)); + r8 = (word64)((r8 >> 9) | (r9 << 55)); + r9 = (word64)((r9 >> 9) | (rax << 55)); + WC_S64(rdi, 32) = (word64)(r11); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rdx = (word64)(WC_L64(rcx, 72)); + rax = (word64)((rax >> 9) | (rdx << 55)); + rdx = (word64)(rdx >> 9); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rdx); + r15 = (word64)(WC_L64(rdi, 64)); + r15 = (word64)(r15 >> 9); + r15 = (word64)(0 - r15); +#ifdef _WIN64 + rdx = (word64)(rsi); + rcx = (word64)(r15); +#else + rcx = (word64)(r15); + rdx = (word64)(rsi); +#endif /* _WIN64 */ + rsi = (word64)(rdi); + (void)sp_521_cond_sub_9((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx, (sp_digit)rcx); +} + +/* Add two Montgomery form numbers (r = a + b % m). + * + * @param [out] r Result of addition. + * @param [in] a First number to add in Montgomery form. + * @param [in] b Second number to add in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_521_mont_add_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12, r13, r14, r15; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rsi, 32)); + r11 = (word64)(WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rsi, 48)); + r13 = (word64)(WC_L64(rsi, 56)); + r14 = (word64)(WC_L64(rsi, 64)); + cf = _addcarry_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 8), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 16), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 24), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 32), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, WC_L64(rdx, 40), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, WC_L64(rdx, 48), (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, WC_L64(rdx, 56), (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, WC_L64(rdx, 64), (unsigned long long*)&r14); + r15 = (word64)(r14); + r14 = (word64)(r14 & 0x1ff); + r15 = (word64)(r15 >> 9); + cf = _addcarry_u64(0, rax, r15, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(r11); + WC_S64(rdi, 48) = (word64)(r12); + WC_S64(rdi, 56) = (word64)(r13); + WC_S64(rdi, 64) = (word64)(r14); + (void)m; +} + +/* Double a Montgomery form number (r = a + a % m). + * + * @param [out] r Result of addition. + * @param [in] a Number to double in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_521_mont_dbl_9(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12, r13, r14; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)(WC_L64(rsi, 40)); + r11 = (word64)(WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rsi, 56)); + r13 = (word64)(WC_L64(rsi, 64)); + cf = _addcarry_u64(0, rdx, rdx, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, rax, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, rcx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + r14 = (word64)(r13); + r13 = (word64)(r13 & 0x1ff); + r14 = (word64)(r14 >> 9); + cf = _addcarry_u64(0, rdx, r14, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); + WC_S64(rdi, 48) = (word64)(r11); + WC_S64(rdi, 56) = (word64)(r12); + WC_S64(rdi, 64) = (word64)(r13); + (void)m; +} + +/* Triple a Montgomery form number (r = a + a + a % m). + * + * @param [out] r Result of Tripling. + * @param [in] a Number to triple in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_521_mont_tpl_9(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12, r13, r14; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)(WC_L64(rsi, 40)); + r11 = (word64)(WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rsi, 56)); + r13 = (word64)(WC_L64(rsi, 64)); + cf = _addcarry_u64(0, rdx, rdx, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, rax, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, rcx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r8, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r9, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, r13, (unsigned long long*)&r13); + cf = _addcarry_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 8), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 16), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 24), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 32), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rsi, 40), (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, WC_L64(rsi, 48), (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, WC_L64(rsi, 56), (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, WC_L64(rsi, 64), (unsigned long long*)&r13); + r14 = (word64)(r13); + r13 = (word64)(r13 & 0x1ff); + r14 = (word64)(r14 >> 9); + cf = _addcarry_u64(0, rdx, r14, (unsigned long long*)&rdx); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r13, 0, (unsigned long long*)&r13); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); + WC_S64(rdi, 48) = (word64)(r11); + WC_S64(rdi, 56) = (word64)(r12); + WC_S64(rdi, 64) = (word64)(r13); + (void)m; +} + +/* Subtract two Montgomery form numbers (r = a - b % m). + * + * @param [out] r Result of addition. + * @param [in] a First number to add in Montgomery form. + * @param [in] b Second number to add in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_521_mont_sub_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12, r13, r14, r15; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rsi, 32)); + r11 = (word64)(WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rsi, 48)); + r13 = (word64)(WC_L64(rsi, 56)); + r14 = (word64)(WC_L64(rsi, 64)); + cf = _subborrow_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rdx, 8), (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 16), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 24), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 32), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, WC_L64(rdx, 40), (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, WC_L64(rdx, 48), (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, WC_L64(rdx, 56), (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, WC_L64(rdx, 64), (unsigned long long*)&r14); + r15 = (word64)(r14); + r14 = (word64)(r14 & 0x1ff); + r15 = (word64)((word64)((sword64)r15 >> 9)); + r15 = (word64)(0 - r15); + cf = _subborrow_u64(0, rax, r15, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + cf = _subborrow_u64(cf, r14, 0, (unsigned long long*)&r14); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(r11); + WC_S64(rdi, 48) = (word64)(r12); + WC_S64(rdi, 56) = (word64)(r13); + WC_S64(rdi, 64) = (word64)(r14); + (void)m; +} + +/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_521_mont_div2_9(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12, r13, r14; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)(WC_L64(rsi, 40)); + r11 = (word64)(WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rsi, 56)); + r13 = (word64)(WC_L64(rsi, 64)); + r14 = (word64)(rdx); + r14 = (word64)(r14 & 1); + cf = _subborrow_u64(0, rdx, r14, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + r14 = (word64)(r14 << 9); + r13 = (word64)(r13 + r14); + rdx = (word64)((rdx >> 1) | (rax << 63)); + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)((r10 >> 1) | (r11 << 63)); + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)(r13 >> 1); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); + WC_S64(rdi, 48) = (word64)(r11); + WC_S64(rdi, 56) = (word64)(r12); + WC_S64(rdi, 64) = (word64)(r13); + (void)m; +} + +#ifndef WC_NO_CACHE_RESISTANT +/* Touch each possible point that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of point to retrieve. + */ +WC_X64I_TARGET("avx") +WOLFSSL_LOCAL void sp_521_get_point_33_9(sp_point_521* r, + const sp_point_521* table, int idx) +{ + word64 rdi, rsi, rdx, r12, rax, r10, r11, rcx = 0, r8 = 0, r9 = 0; + __m128i x0, x1, x2, x3, x4, x5, x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x8 = _mm_setzero_si128(), + x9 = _mm_setzero_si128(), x10 = _mm_setzero_si128(), + x11 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), x13, x14, + x15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + r12 = (word64)(1); + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); +#ifndef SP_ALIGN_16 + rsi = (word64)(rsi + 0x1b8); +#else + rsi = (word64)(rsi + 0x1c0); +#endif /* !SP_ALIGN_16 */ + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x20); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + x4 = _mm_setzero_si128(); + x5 = _mm_setzero_si128(); + r10 = (word64)(0); + r11 = (word64)(0); + x14 = x15; +L_521_get_point_33_9_start_1: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + rcx = (word64)(0); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((rdx) == (r12)) & 0xff); + rcx = (word64)(0 - rcx); + r12 = (word64)(r12 + 1); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + r8 = (word64)(WC_L64(rsi, 64)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 144)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 160)); +#ifndef SP_ALIGN_16 + rsi = (word64)(rsi + 0x1b8); +#else + rsi = (word64)(rsi + 0x1c0); +#endif /* !SP_ALIGN_16 */ + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + x8 = _mm_and_si128(x8, x12); + x9 = _mm_and_si128(x9, x12); + x10 = _mm_and_si128(x10, x12); + x11 = _mm_and_si128(x11, x12); + r8 = (word64)(r8 & rcx); + x0 = _mm_or_si128(x0, x6); + x1 = _mm_or_si128(x1, x7); + x2 = _mm_or_si128(x2, x8); + x3 = _mm_or_si128(x3, x9); + x4 = _mm_or_si128(x4, x10); + x5 = _mm_or_si128(x5, x11); + r10 = (word64)(r10 | r8); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_521_get_point_33_9_start_1; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 48), x3); + WC_S64(rdi, 64) = (word64)(r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 144), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 160), x5); + r12 = (word64)(1); + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); +#ifndef SP_ALIGN_16 + rsi = (word64)(rsi - 0x3700); +#else + rsi = (word64)(rsi - 0x3800); +#endif /* !SP_ALIGN_16 */ + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x20); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + x4 = _mm_setzero_si128(); + x5 = _mm_setzero_si128(); + r10 = (word64)(0); + r11 = (word64)(0); + x14 = x15; +L_521_get_point_33_9_start_2: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + rcx = (word64)(0); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((rdx) == (r12)) & 0xff); + rcx = (word64)(0 - rcx); + r12 = (word64)(r12 + 1); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 176)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 192)); + r8 = (word64)(WC_L64(rsi, 208)); + x8 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 288)); + x9 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 304)); + x10 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 320)); + x11 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 336)); + r9 = (word64)(WC_L64(rsi, 352)); +#ifndef SP_ALIGN_16 + rsi = (word64)(rsi + 0x1b8); +#else + rsi = (word64)(rsi + 0x1c0); +#endif /* !SP_ALIGN_16 */ + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + x8 = _mm_and_si128(x8, x12); + x9 = _mm_and_si128(x9, x12); + x10 = _mm_and_si128(x10, x12); + x11 = _mm_and_si128(x11, x12); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + x0 = _mm_or_si128(x0, x6); + x1 = _mm_or_si128(x1, x7); + x2 = _mm_or_si128(x2, x8); + x3 = _mm_or_si128(x3, x9); + x4 = _mm_or_si128(x4, x10); + x5 = _mm_or_si128(x5, x11); + r10 = (word64)(r10 | r8); + r11 = (word64)(r11 | r9); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_521_get_point_33_9_start_2; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 176), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 192), x1); + WC_S64(rdi, 208) = (word64)(r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 288), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 304), x3); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 320), x4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 336), x5); + WC_S64(rdi, 352) = (word64)(r11); +} + +#ifdef HAVE_INTEL_AVX2 +/* Touch each possible point that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of point to retrieve. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_521_get_point_33_avx2_9(sp_point_521* r, + const sp_point_521* table, int idx) +{ + word64 rdi, rsi, rdx, r14, rax, r8, r9, r10, rcx = 0, r11 = 0, r12 = 0, + r13 = 0; + __m256i y0, y1, y2, y3, y4, y5, y6 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(), y8 = _mm256_setzero_si256(), + y9 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(), + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), y13, + y14, y15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + r14 = (word64)(1); + rax = (word64)(1); + y13 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); +#ifndef SP_ALIGN_16 + rsi = (word64)(rsi + 0x1b8); +#else + rsi = (word64)(rsi + 0x1c0); +#endif /* !SP_ALIGN_16 */ + y15 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + rax = (word64)(0x20); + y14 = _mm256_setzero_si256(); + y13 = _mm256_permutevar8x32_epi32(y13, y14); + y15 = _mm256_permutevar8x32_epi32(y15, y14); + y0 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + r8 = (word64)(0); + r9 = (word64)(0); + r10 = (word64)(0); + y14 = y15; +L_521_get_point_33_avx2_9_start: + y12 = _mm256_cmpeq_epi32(y14, y13); + y14 = _mm256_add_epi32(y14, y15); + rcx = (word64)(0); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((rdx) == (r14)) & 0xff); + rcx = (word64)(0 - rcx); + r14 = (word64)(r14 + 1); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 144)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 176)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + r11 = (word64)(WC_L64(rsi, 64)); + r12 = (word64)(WC_L64(rsi, 208)); + r13 = (word64)(WC_L64(rsi, 352)); +#ifndef SP_ALIGN_16 + rsi = (word64)(rsi + 0x1b8); +#else + rsi = (word64)(rsi + 0x1c0); +#endif /* !SP_ALIGN_16 */ + y6 = _mm256_and_si256(y6, y12); + y7 = _mm256_and_si256(y7, y12); + y8 = _mm256_and_si256(y8, y12); + y9 = _mm256_and_si256(y9, y12); + y10 = _mm256_and_si256(y10, y12); + y11 = _mm256_and_si256(y11, y12); + r11 = (word64)(r11 & rcx); + r12 = (word64)(r12 & rcx); + r13 = (word64)(r13 & rcx); + y0 = _mm256_or_si256(y0, y6); + y1 = _mm256_or_si256(y1, y7); + y2 = _mm256_or_si256(y2, y8); + y3 = _mm256_or_si256(y3, y9); + y4 = _mm256_or_si256(y4, y10); + y5 = _mm256_or_si256(y5, y11); + r8 = (word64)(r8 | r11); + r9 = (word64)(r9 | r12); + r10 = (word64)(r10 | r13); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_521_get_point_33_avx2_9_start; + } + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 144), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 176), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y5); + WC_S64(rdi, 64) = (word64)(r8); + WC_S64(rdi, 208) = (word64)(r9); + WC_S64(rdi, 352) = (word64)(r10); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WC_NO_CACHE_RESISTANT */ +#ifdef HAVE_INTEL_AVX2 +/* Multiply two Montgomery form numbers mod the modulus (prime). + * (r = a * b mod m) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply in Montgomery form. + * @param [in] b Second number to multiply in Montgomery form. + * @param [in] m Modulus (prime). + * @param [in] mp Montgomery multiplier. + */ +WOLFSSL_LOCAL void sp_521_mont_mul_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m, sp_digit mp) +{ + sp_521_mont_mul_9(r, a, b, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Square the Montgomery form number mod the modulus (prime). (r = a * a mod m) + * + * @param [out] r Result of squaring. + * @param [in] a Number to square in Montgomery form. + * @param [in] m Modulus (prime). + * @param [in] mp Montgomery multiplier. + */ +WOLFSSL_LOCAL void sp_521_mont_sqr_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit* m, sp_digit mp) +{ + sp_521_mont_sqr_9(r, a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_521_cond_sub_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r10, r8, r9, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _subborrow_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(r10); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 521 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_521_mont_reduce_order_avx2_9(sp_digit* a, + const sp_digit* m, sp_digit mp) +{ + sp_521_mont_reduce_order_9(a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_521_mont_div2_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10, r11, r12, r13, r14; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)(WC_L64(rsi, 40)); + r11 = (word64)(WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rsi, 56)); + r13 = (word64)(WC_L64(rsi, 64)); + r14 = (word64)(rdx); + r14 = (word64)(r14 & 1); + cf = _subborrow_u64(0, rdx, r14, (unsigned long long*)&rdx); + cf = _subborrow_u64(cf, rax, 0, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _subborrow_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _subborrow_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _subborrow_u64(cf, r13, 0, (unsigned long long*)&r13); + r14 = (word64)(r14 << 9); + r13 = (word64)(r13 + r14); + rdx = (word64)((rdx >> 1) | (rax << 63)); + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + r10 = (word64)((r10 >> 1) | (r11 << 63)); + r11 = (word64)((r11 >> 1) | (r12 << 63)); + r12 = (word64)((r12 >> 1) | (r13 << 63)); + r13 = (word64)(r13 >> 1); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r9); + WC_S64(rdi, 40) = (word64)(r10); + WC_S64(rdi, 48) = (word64)(r11); + WC_S64(rdi, 56) = (word64)(r12); + WC_S64(rdi, 64) = (word64)(r13); + (void)m; +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifndef WC_NO_CACHE_RESISTANT +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL void sp_521_get_entry_64_9(sp_point_521* r, + const sp_table_entry_521* table, int idx) +{ + word64 rdi, rsi, rdx, r10, rax, r9, rcx = 0, r8 = 0; + __m128i x0, x1, x2, x3, x4 = _mm_setzero_si128(), + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), x13, x14, + x15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + /* From entry 1 */ + r10 = (word64)(1); + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi + 0x90); + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x3f); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + r9 = (word64)(0); + x14 = x15; +L_521_get_entry_64_9_start_0: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + rcx = (word64)(0); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((rdx) == (r10)) & 0xff); + rcx = (word64)(0 - rcx); + r10 = (word64)(r10 + 1); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + r8 = (word64)(WC_L64(rsi, 64)); + rsi = (word64)(rsi + 0x90); + x4 = _mm_and_si128(x4, x12); + x5 = _mm_and_si128(x5, x12); + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + r8 = (word64)(r8 & rcx); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + r9 = (word64)(r9 | r8); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_521_get_entry_64_9_start_0; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 48), x3); + WC_S64(rdi, 64) = (word64)(r9); + /* From entry 1 */ + r10 = (word64)(1); + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi - 0x2328); + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x3f); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + r9 = (word64)(0); + x14 = x15; +L_521_get_entry_64_9_start_1: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + rcx = (word64)(0); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((rdx) == (r10)) & 0xff); + rcx = (word64)(0 - rcx); + r10 = (word64)(r10 + 1); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + r8 = (word64)(WC_L64(rsi, 64)); + rsi = (word64)(rsi + 0x90); + x4 = _mm_and_si128(x4, x12); + x5 = _mm_and_si128(x5, x12); + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + r8 = (word64)(r8 & rcx); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + r9 = (word64)(r9 | r8); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_521_get_entry_64_9_start_1; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 144), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 160), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 176), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 192), x3); + WC_S64(rdi, 208) = (word64)(r9); +} + +#ifdef HAVE_INTEL_AVX2 +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_521_get_entry_64_avx2_9(sp_point_521* r, + const sp_table_entry_521* table, int idx) +{ + word64 rdi, rsi, rdx, r12, rax, r8, r9, rcx = 0, r10 = 0, r11 = 0; + __m256i y0, y1, y2, y3, y4 = _mm256_setzero_si256(), + y5 = _mm256_setzero_si256(), y6 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(), y8 = _mm256_setzero_si256(), y9, y10, + y11; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + r12 = (word64)(1); + rax = (word64)(1); + y9 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + rsi = (word64)(rsi + 0x90); + y11 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + rax = (word64)(0x40); + y10 = _mm256_setzero_si256(); + y9 = _mm256_permutevar8x32_epi32(y9, y10); + y11 = _mm256_permutevar8x32_epi32(y11, y10); + y0 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + r8 = (word64)(0); + r9 = (word64)(0); + y10 = y11; +L_521_get_entry_64_avx2_9_start: + y8 = _mm256_cmpeq_epi32(y10, y9); + y10 = _mm256_add_epi32(y10, y11); + rcx = (word64)(0); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((rdx) == (r12)) & 0xff); + rcx = (word64)(0 - rcx); + r12 = (word64)(r12 + 1); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 104)); + r10 = (word64)(WC_L64(rsi, 64)); + r11 = (word64)(WC_L64(rsi, 136)); + rsi = (word64)(rsi + 0x90); + y4 = _mm256_and_si256(y4, y8); + y5 = _mm256_and_si256(y5, y8); + y6 = _mm256_and_si256(y6, y8); + y7 = _mm256_and_si256(y7, y8); + r10 = (word64)(r10 & rcx); + r11 = (word64)(r11 & rcx); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + r8 = (word64)(r8 | r10); + r9 = (word64)(r9 | r11); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_521_get_entry_64_avx2_9_start; + } + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 144), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 176), y3); + WC_S64(rdi, 64) = (word64)(r8); + WC_S64(rdi, 208) = (word64)(r9); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WC_NO_CACHE_RESISTANT */ +#ifndef WC_NO_CACHE_RESISTANT +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL void sp_521_get_entry_65_9(sp_point_521* r, + const sp_table_entry_521* table, int idx) +{ + word64 rdi, rsi, rdx, r10, rax, r9, rcx = 0, r8 = 0; + __m128i x0, x1, x2, x3, x4 = _mm_setzero_si128(), + x5 = _mm_setzero_si128(), x6 = _mm_setzero_si128(), + x7 = _mm_setzero_si128(), x12 = _mm_setzero_si128(), x13, x14, + x15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + /* From entry 1 */ + r10 = (word64)(1); + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi + 0x90); + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x40); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + r9 = (word64)(0); + x14 = x15; +L_521_get_entry_65_9_start_0: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + rcx = (word64)(0); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((rdx) == (r10)) & 0xff); + rcx = (word64)(0 - rcx); + r10 = (word64)(r10 + 1); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + r8 = (word64)(WC_L64(rsi, 64)); + rsi = (word64)(rsi + 0x90); + x4 = _mm_and_si128(x4, x12); + x5 = _mm_and_si128(x5, x12); + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + r8 = (word64)(r8 & rcx); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + r9 = (word64)(r9 | r8); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_521_get_entry_65_9_start_0; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 48), x3); + WC_S64(rdi, 64) = (word64)(r9); + /* From entry 1 */ + r10 = (word64)(1); + rax = (word64)(1); + x13 = _mm_cvtsi32_si128((int)(word32)rdx); + rsi = (word64)(rsi - 0x23b8); + x15 = _mm_cvtsi32_si128((int)(word32)rax); + rax = (word64)(0x40); + x15 = _mm_shuffle_epi32(x15, 0); + x13 = _mm_shuffle_epi32(x13, 0); + x14 = _mm_setzero_si128(); + x0 = _mm_setzero_si128(); + x1 = _mm_setzero_si128(); + x2 = _mm_setzero_si128(); + x3 = _mm_setzero_si128(); + r9 = (word64)(0); + x14 = x15; +L_521_get_entry_65_9_start_1: + x12 = x14; + x14 = _mm_add_epi32(x14, x15); + x12 = _mm_cmpeq_epi32(x12, x13); + rcx = (word64)(0); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((rdx) == (r10)) & 0xff); + rcx = (word64)(0 - rcx); + r10 = (word64)(r10 + 1); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 0)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 16)); + x6 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 32)); + x7 = _mm_loadu_si128((const __m128i*)WC_PR(rsi, 48)); + r8 = (word64)(WC_L64(rsi, 64)); + rsi = (word64)(rsi + 0x90); + x4 = _mm_and_si128(x4, x12); + x5 = _mm_and_si128(x5, x12); + x6 = _mm_and_si128(x6, x12); + x7 = _mm_and_si128(x7, x12); + r8 = (word64)(r8 & rcx); + x0 = _mm_or_si128(x0, x4); + x1 = _mm_or_si128(x1, x5); + x2 = _mm_or_si128(x2, x6); + x3 = _mm_or_si128(x3, x7); + r9 = (word64)(r9 | r8); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_521_get_entry_65_9_start_1; + } + _mm_storeu_si128((__m128i*)WC_PW(rdi, 144), x0); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 160), x1); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 176), x2); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 192), x3); + WC_S64(rdi, 208) = (word64)(r9); +} + +#ifdef HAVE_INTEL_AVX2 +/* Touch each possible entry that could be being copied. + * + * @param [out] r Point to copy into. + * @param [in] table Table - start of the entries to access + * @param [in] idx Index of entry to retrieve. + */ +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void sp_521_get_entry_65_avx2_9(sp_point_521* r, + const sp_table_entry_521* table, int idx) +{ + word64 rdi, rsi, rdx, r12, rax, r8, r9, rcx = 0, r10 = 0, r11 = 0; + __m256i y0, y1, y2, y3, y4 = _mm256_setzero_si256(), + y5 = _mm256_setzero_si256(), y6 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(), y8 = _mm256_setzero_si256(), y9, y10, + y11; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)table; + rdx = (word64)(word64)idx; + + r12 = (word64)(1); + rax = (word64)(1); + y9 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + rsi = (word64)(rsi + 0x90); + y11 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + rax = (word64)(0x41); + y10 = _mm256_setzero_si256(); + y9 = _mm256_permutevar8x32_epi32(y9, y10); + y11 = _mm256_permutevar8x32_epi32(y11, y10); + y0 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + r8 = (word64)(0); + r9 = (word64)(0); + y10 = y11; +L_521_get_entry_65_avx2_9_start: + y8 = _mm256_cmpeq_epi32(y10, y9); + y10 = _mm256_add_epi32(y10, y11); + rcx = (word64)(0); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((rdx) == (r12)) & 0xff); + rcx = (word64)(0 - rcx); + r12 = (word64)(r12 + 1); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 104)); + r10 = (word64)(WC_L64(rsi, 64)); + r11 = (word64)(WC_L64(rsi, 136)); + rsi = (word64)(rsi + 0x90); + y4 = _mm256_and_si256(y4, y8); + y5 = _mm256_and_si256(y5, y8); + y6 = _mm256_and_si256(y6, y8); + y7 = _mm256_and_si256(y7, y8); + r10 = (word64)(r10 & rcx); + r11 = (word64)(r11 & rcx); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + r8 = (word64)(r8 | r10); + r9 = (word64)(r9 | r11); + rax = (word64)(rax - 1); + if ((rax) != (0)) { + goto L_521_get_entry_65_avx2_9_start; + } + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 144), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 176), y3); + WC_S64(rdi, 64) = (word64)(r8); + WC_S64(rdi, 208) = (word64)(r9); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* !WC_NO_CACHE_RESISTANT */ +/* Add 1 to a. (a = a + 1) + * + * @param [in, out] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_521_add_one_9(sp_digit* a) +{ + word64 rdi; + unsigned char cf; + + rdi = (word64)(size_t)a; + + cf = _addcarry_u64(0, WC_L64(rdi, 0), 1, (unsigned long long*)&WC_S64(rdi, + 0)); + cf = _addcarry_u64(cf, WC_L64(rdi, 8), 0, (unsigned long long*)&WC_S64(rdi, + 8)); + cf = _addcarry_u64(cf, WC_L64(rdi, 16), 0, (unsigned long long*)&WC_S64(rdi, + 16)); + cf = _addcarry_u64(cf, WC_L64(rdi, 24), 0, (unsigned long long*)&WC_S64(rdi, + 24)); + cf = _addcarry_u64(cf, WC_L64(rdi, 32), 0, (unsigned long long*)&WC_S64(rdi, + 32)); + cf = _addcarry_u64(cf, WC_L64(rdi, 40), 0, (unsigned long long*)&WC_S64(rdi, + 40)); + cf = _addcarry_u64(cf, WC_L64(rdi, 48), 0, (unsigned long long*)&WC_S64(rdi, + 48)); + cf = _addcarry_u64(cf, WC_L64(rdi, 56), 0, (unsigned long long*)&WC_S64(rdi, + 56)); + cf = _addcarry_u64(cf, WC_L64(rdi, 64), 0, (unsigned long long*)&WC_S64(rdi, + 64)); +} + +/* Read big endian unsigned byte array into r. + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WOLFSSL_LOCAL void sp_521_from_bin_bswap(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, r11, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x41); + r11 = (word64)(0); + goto L_521_from_bin_bswap_64_end; +L_521_from_bin_bswap_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_L64(r9, 56)); + r8 = (word64)(WC_L64(r9, 48)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_L64(r9, 40)); + r8 = (word64)(WC_L64(r9, 32)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(r9, 24)); + r8 = (word64)(WC_L64(r9, 16)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_L64(r9, 8)); + r8 = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_521_from_bin_bswap_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_521_from_bin_bswap_64_start; + } + goto L_521_from_bin_bswap_8_end; +L_521_from_bin_bswap_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_521_from_bin_bswap_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_521_from_bin_bswap_8_start; + } + if ((rcx) == (r11)) { + goto L_521_from_bin_bswap_hi_end; + } + r8 = (word64)(r11); + rax = (word64)(r11); +L_521_from_bin_bswap_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_521_from_bin_bswap_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_521_from_bin_bswap_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_521_from_bin_bswap_zero_end; + } +L_521_from_bin_bswap_zero_start: + WC_S64(rdi, 0) = (word64)(r11); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_521_from_bin_bswap_zero_start; + } +L_521_from_bin_bswap_zero_end: + ; + (void)rsi; +} + +#ifndef NO_MOVBE_SUPPORT +/* Read big endian unsigned byte array into r. + * Uses the movbe instruction which is an optional instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_521_from_bin_movbe(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x42); + goto L_521_from_bin_movbe_64_end; +L_521_from_bin_movbe_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 56))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 48))); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 40))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 32))); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 24))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 16))); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 8))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_521_from_bin_movbe_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_521_from_bin_movbe_64_start; + } + goto L_521_from_bin_movbe_8_end; +L_521_from_bin_movbe_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_521_from_bin_movbe_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_521_from_bin_movbe_8_start; + } + if ((rcx) == (0)) { + goto L_521_from_bin_movbe_hi_end; + } + r8 = (word64)(0); + rax = (word64)(0); +L_521_from_bin_movbe_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_521_from_bin_movbe_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_521_from_bin_movbe_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_521_from_bin_movbe_zero_end; + } +L_521_from_bin_movbe_zero_start: + WC_S64(rdi, 0) = (word64)(0); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_521_from_bin_movbe_zero_start; + } +L_521_from_bin_movbe_zero_end: + ; + (void)rsi; +} + +#endif /* !NO_MOVBE_SUPPORT */ +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 65 + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WOLFSSL_LOCAL void sp_521_to_bin_bswap_9(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rax = 0, rdx = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, 64)) & 0xff); + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, 65)) & 0xff); + WC_S8(rsi, 0) = (byte)((byte)rdx); + WC_S8(rsi, 1) = (byte)((byte)rax); + rdx = (word64)(WC_L64(rdi, 56)); + rax = (word64)(WC_L64(rdi, 48)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 2) = (word64)(rdx); + WC_S64(rsi, 10) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 40)); + rax = (word64)(WC_L64(rdi, 32)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 18) = (word64)(rdx); + WC_S64(rsi, 26) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 24)); + rax = (word64)(WC_L64(rdi, 16)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 34) = (word64)(rdx); + WC_S64(rsi, 42) = (word64)(rax); + rdx = (word64)(WC_L64(rdi, 8)); + rax = (word64)(WC_L64(rdi, 0)); + rdx = (word64)(WC_X64I_BSWAP64(rdx)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rsi, 50) = (word64)(rdx); + WC_S64(rsi, 58) = (word64)(rax); +} + +#ifndef NO_MOVBE_SUPPORT +/* Write r as big endian to byte array. + * Fixed length number of bytes written: 65 + * Uses the movbe instruction which is optional. + * + * @param [out] r A single precision integer. + * @param [in] a Byte array. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_521_to_bin_movbe_9(sp_digit* r, byte* a) +{ + word64 rdi, rsi, rax = 0, rdx = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, 64)) & 0xff); + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdi, 65)) & 0xff); + WC_S8(rsi, 0) = (byte)((byte)rdx); + WC_S8(rsi, 1) = (byte)((byte)rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 56))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 48))); + WC_S64(rsi, 2) = (word64)(rdx); + WC_S64(rsi, 10) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 40))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 32))); + WC_S64(rsi, 18) = (word64)(rdx); + WC_S64(rsi, 26) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 24))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 16))); + WC_S64(rsi, 34) = (word64)(rdx); + WC_S64(rsi, 42) = (word64)(rax); + rdx = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 8))); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(rdi, 0))); + WC_S64(rsi, 50) = (word64)(rdx); + WC_S64(rsi, 58) = (word64)(rax); +} + +#endif /* NO_MOVBE_SUPPORT */ +/* Shift number right by 1 bit. (r = a >> 1) + * + * @param [out] r Result of right shift by 1. + * @param [in] a Number to shift. + */ +WOLFSSL_LOCAL void sp_521_rshift_9(sp_digit* r, const sp_digit* a, int n) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)n; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)((rdx >> ((byte)rcx & 63)) | ((rax << 1) << (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax >> ((byte)rcx & 63)) | ((r8 << 1) << (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 >> ((byte)rcx & 63)) | ((r9 << 1) << (63 - (( + byte)rcx & 63)))); + r9 = (word64)((r9 >> ((byte)rcx & 63)) | ((r10 << 1) << (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsi, 64)); + r10 = (word64)((r10 >> ((byte)rcx & 63)) | ((rax << 1) << (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax >> ((byte)rcx & 63)) | ((r8 << 1) << (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 >> ((byte)rcx & 63)) | ((r9 << 1) << (63 - (( + byte)rcx & 63)))); + r9 = (word64)((r9 >> ((byte)rcx & 63)) | ((rdx << 1) << (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(rax); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rdx = (word64)(rdx >> ((byte)rcx & 63)); + WC_S64(rdi, 64) = (word64)(rdx); +} + +/* Shift number left by n bit. (r = a << n) + * + * @param [out] r Result of left shift by n. + * @param [in] a Number to shift. + * @param [in] n Amoutnt o shift. + */ +WOLFSSL_LOCAL void sp_521_lshift_9(sp_digit* r, const sp_digit* a, int n) +{ + word64 rdi, rsi, rcx, r10, r11, rdx, rax, r8, r9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r10 = (word64)(0); + r11 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsi, 40)); + rax = (word64)(WC_L64(rsi, 48)); + r8 = (word64)(WC_L64(rsi, 56)); + r9 = (word64)(WC_L64(rsi, 64)); + r10 = (word64)((r10 << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + WC_S64(rdi, 64) = (word64)(r9); + WC_S64(rdi, 72) = (word64)(r10); + r9 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsi, 8)); + rax = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + WC_S64(rdi, 32) = (word64)(r11); + r9 = (word64)(r9 << ((byte)rcx & 63)); + WC_S64(rdi, 0) = (word64)(r9); +} + +/* Shift number left by n bit. (r = a << n) + * + * @param [out] r Result of left shift by n. + * @param [in] a Number to shift. + * @param [in] n Amoutnt o shift. + */ +WOLFSSL_LOCAL void sp_521_lshift_18(sp_digit* r, const sp_digit* a, int n) +{ + word64 rdi, rsi, rcx, r10, r11, rdx, rax, r8, r9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r10 = (word64)(0); + r11 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsi, 112)); + rax = (word64)(WC_L64(rsi, 120)); + r8 = (word64)(WC_L64(rsi, 128)); + r9 = (word64)(WC_L64(rsi, 136)); + r10 = (word64)((r10 << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 112) = (word64)(rdx); + WC_S64(rdi, 120) = (word64)(rax); + WC_S64(rdi, 128) = (word64)(r8); + WC_S64(rdi, 136) = (word64)(r9); + WC_S64(rdi, 144) = (word64)(r10); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsi, 80)); + rax = (word64)(WC_L64(rsi, 88)); + r8 = (word64)(WC_L64(rsi, 96)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 80) = (word64)(rdx); + WC_S64(rdi, 88) = (word64)(rax); + WC_S64(rdi, 96) = (word64)(r8); + WC_S64(rdi, 104) = (word64)(r11); + r11 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsi, 48)); + rax = (word64)(WC_L64(rsi, 56)); + r8 = (word64)(WC_L64(rsi, 64)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r11 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 48) = (word64)(rdx); + WC_S64(rdi, 56) = (word64)(rax); + WC_S64(rdi, 64) = (word64)(r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsi, 16)); + rax = (word64)(WC_L64(rsi, 24)); + r8 = (word64)(WC_L64(rsi, 32)); + r11 = (word64)((r11 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)((r8 << ((byte)rcx & 63)) | ((rax >> 1) >> (63 - (( + byte)rcx & 63)))); + rax = (word64)((rax << ((byte)rcx & 63)) | ((rdx >> 1) >> (63 - (( + byte)rcx & 63)))); + rdx = (word64)((rdx << ((byte)rcx & 63)) | ((r9 >> 1) >> (63 - (( + byte)rcx & 63)))); + WC_S64(rdi, 16) = (word64)(rdx); + WC_S64(rdi, 24) = (word64)(rax); + WC_S64(rdi, 32) = (word64)(r8); + WC_S64(rdi, 40) = (word64)(r11); + r8 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)((r9 << ((byte)rcx & 63)) | ((r8 >> 1) >> (63 - (( + byte)rcx & 63)))); + r8 = (word64)(r8 << ((byte)rcx & 63)); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); +} + +/* Sub b from a into a. (a -= b) + * + * @param [in, out] a A single precision integer and result. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_521_sub_in_place_9(sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rdi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 16), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 24), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 32), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 48), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 56), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 64), (unsigned long long*)&rdx); + WC_S64(rdi, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_521_mul_d_9(sp_digit* r, const sp_digit* a, sp_digit b) +{ + word64 rdi, rsi, rcx, rax, r10, rdx = 0, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)b; + + /* A[0] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + WC_S64(rdi, 0) = (word64)(r8); + /* A[1] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 64) = (word64)(r10); + WC_S64(rdi, 72) = (word64)(r8); +} + +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_521_mul_d_avx2_9(sp_digit* r, const sp_digit* a, + const sp_digit b) +{ + sp_521_mul_d_9(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef _WIN64 +/* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) + * + * @param [in] d1 The high order half of the number to divide. + * @param [in] d0 The low order half of the number to divide. + * @param [in] div The dividend. + * + * @return The result of the division. + */ +WOLFSSL_LOCAL sp_digit div_521_word_asm_9(sp_digit d1, sp_digit d0, + sp_digit div) +{ + word64 rdi, rsi, rcx, rax, rdx; + + rdi = (word64)(word64)d1; + rsi = (word64)(word64)d0; + rcx = (word64)(word64)div; + + rax = (word64)(rsi); + rdx = (word64)(rdi); + WC_X64I_DIV128(rax, rdx, rdx, rax, rcx); + return (sp_digit)(word64)rax; +} + +#endif /* _WIN64 */ +/* Shift number right by 1 bit. (r = a >> 1) + * + * @param [out] r Result of right shift by 1. + * @param [in] a Number to shift. + */ +WOLFSSL_LOCAL void sp_521_rshift1_9(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r10; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rdx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(WC_L64(rsi, 8)); + rcx = (word64)(WC_L64(rsi, 16)); + r8 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)((rdx >> 1) | (rax << 63)); + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r10 << 63)); + WC_S64(rdi, 0) = (word64)(rdx); + WC_S64(rdi, 8) = (word64)(rax); + WC_S64(rdi, 16) = (word64)(rcx); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(rsi, 40)); + rcx = (word64)(WC_L64(rsi, 48)); + r8 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsi, 64)); + r10 = (word64)((r10 >> 1) | (rax << 63)); + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (rdx << 63)); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(rax); + WC_S64(rdi, 48) = (word64)(rcx); + WC_S64(rdi, 56) = (word64)(r8); + rdx = (word64)(rdx >> 1); + WC_S64(rdi, 64) = (word64)(rdx); +} + +/* Divide the number by 2 mod the prime. (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus + */ +WOLFSSL_LOCAL void sp_521_div2_mod_9(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx = 0, r8 = 0, r9 = 0, r10 = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rax = (word64)(WC_L64(rsi, 0)); + rax = (word64)(rax & 1); + if ((rax) == (0)) { + goto L_521_mod_inv_9_div2_mod_no_add; + } + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + cf = _addcarry_u64(0, rax, r8, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + rax = (word64)(WC_L64(rsi, 16)); + rcx = (word64)(WC_L64(rsi, 24)); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + cf = _addcarry_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(rcx); + rax = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + cf = _addcarry_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + rax = (word64)(WC_L64(rsi, 48)); + rcx = (word64)(WC_L64(rsi, 56)); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + cf = _addcarry_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(rcx); + rax = (word64)(WC_L64(rsi, 64)); + r8 = (word64)(WC_L64(rdx, 64)); + cf = _addcarry_u64(cf, rax, r8, (unsigned long long*)&rax); + WC_S64(rdi, 64) = (word64)(rax); +L_521_mod_inv_9_div2_mod_no_add: + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + r10 = (word64)(WC_L64(rsi, 32)); + rax = (word64)((rax >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rcx = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + rax = (word64)(WC_L64(rsi, 64)); + r10 = (word64)((r10 >> 1) | (rcx << 63)); + rcx = (word64)((rcx >> 1) | (r8 << 63)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + r9 = (word64)((r9 >> 1) | (rax << 63)); + WC_S64(rdi, 32) = (word64)(r10); + WC_S64(rdi, 40) = (word64)(rcx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(rax >> 1); + WC_S64(rdi, 64) = (word64)(rax); +} + +WOLFSSL_LOCAL int sp_521_num_bits_9(const sp_digit * a) +{ + word64 rdi, rax, rdx; + + rdi = (word64)(size_t)a; + + rax = (word64)(0); + rdx = (word64)(WC_L64(rdi, 64)); + if ((rdx) == (0)) { + goto L_521_num_bits_9_end_512; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x201); + goto L_521_num_bits_9_done; +L_521_num_bits_9_end_512: + rdx = (word64)(WC_L64(rdi, 56)); + if ((rdx) == (0)) { + goto L_521_num_bits_9_end_448; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x1c1); + goto L_521_num_bits_9_done; +L_521_num_bits_9_end_448: + rdx = (word64)(WC_L64(rdi, 48)); + if ((rdx) == (0)) { + goto L_521_num_bits_9_end_384; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x181); + goto L_521_num_bits_9_done; +L_521_num_bits_9_end_384: + rdx = (word64)(WC_L64(rdi, 40)); + if ((rdx) == (0)) { + goto L_521_num_bits_9_end_320; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x141); + goto L_521_num_bits_9_done; +L_521_num_bits_9_end_320: + rdx = (word64)(WC_L64(rdi, 32)); + if ((rdx) == (0)) { + goto L_521_num_bits_9_end_256; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x101); + goto L_521_num_bits_9_done; +L_521_num_bits_9_end_256: + rdx = (word64)(WC_L64(rdi, 24)); + if ((rdx) == (0)) { + goto L_521_num_bits_9_end_192; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0xc1); + goto L_521_num_bits_9_done; +L_521_num_bits_9_end_192: + rdx = (word64)(WC_L64(rdi, 16)); + if ((rdx) == (0)) { + goto L_521_num_bits_9_end_128; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x81); + goto L_521_num_bits_9_done; +L_521_num_bits_9_end_128: + rdx = (word64)(WC_L64(rdi, 8)); + if ((rdx) == (0)) { + goto L_521_num_bits_9_end_64; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 0x41); + goto L_521_num_bits_9_done; +L_521_num_bits_9_end_64: + rdx = (word64)(WC_L64(rdi, 0)); + if ((rdx) == (0)) { + goto L_521_num_bits_9_end_0; + } + rax = (word64)(-1); + rax = (word64)(WC_X64I_BSR64(rdx)); + /* BSR: source is non-zero at this point; a zero source would + leave the destination unchanged rather than land here. */ + rax = (word64)(rax + 1); + goto L_521_num_bits_9_done; +L_521_num_bits_9_end_0: +L_521_num_bits_9_done: + return (int)(word64)rax; +} + +#endif /* WOLFSSL_SP_521 */ +#ifdef WOLFSSL_SP_1024 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL void sp_1024_mul_16(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rcx, rsp, rax, rdx = 0, r10, r9, r8; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(size_t)b; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + /* A[0] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r9 = (word64)(rdx); + /* A[0] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 8) = (word64)(r9); + /* A[0] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 16) = (word64)(r10); + /* A[0] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 24) = (word64)(r8); + /* A[0] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 32) = (word64)(r9); + /* A[0] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 40) = (word64)(r10); + /* A[0] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 48) = (word64)(r8); + /* A[0] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 56) = (word64)(r9); + /* A[0] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 64) = (word64)(r10); + /* A[0] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 72) = (word64)(r8); + /* A[0] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 80) = (word64)(r9); + /* A[0] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 88) = (word64)(r10); + /* A[0] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 96) = (word64)(r8); + /* A[0] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 104) = (word64)(r9); + /* A[0] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[2] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 112) = (word64)(r10); + /* A[0] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[1] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[2] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[3] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[0] */ + rax = (word64)(WC_L64(rcx, 0)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rsp, 120) = (word64)(r8); + /* A[1] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[3] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[4] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[1] */ + rax = (word64)(WC_L64(rcx, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 128) = (word64)(r9); + /* A[2] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[4] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[5] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[2] */ + rax = (word64)(WC_L64(rcx, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 136) = (word64)(r10); + /* A[3] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[5] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[6] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[3] */ + rax = (word64)(WC_L64(rcx, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 144) = (word64)(r8); + /* A[4] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[6] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[7] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[4] */ + rax = (word64)(WC_L64(rcx, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 152) = (word64)(r9); + /* A[5] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[7] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[8] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[5] */ + rax = (word64)(WC_L64(rcx, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 160) = (word64)(r10); + /* A[6] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[8] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[9] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[6] */ + rax = (word64)(WC_L64(rcx, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 168) = (word64)(r8); + /* A[7] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[9] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[10] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[7] */ + rax = (word64)(WC_L64(rcx, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 176) = (word64)(r9); + /* A[8] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[10] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[11] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[8] */ + rax = (word64)(WC_L64(rcx, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 184) = (word64)(r10); + /* A[9] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[11] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[12] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[9] */ + rax = (word64)(WC_L64(rcx, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 192) = (word64)(r8); + /* A[10] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[10] */ + rax = (word64)(WC_L64(rcx, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 200) = (word64)(r9); + /* A[11] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[14] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[11] */ + rax = (word64)(WC_L64(rcx, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 208) = (word64)(r10); + /* A[12] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r10 = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[14] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[15] * B[12] */ + rax = (word64)(WC_L64(rcx, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + WC_S64(rdi, 216) = (word64)(r8); + /* A[13] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[15] * B[13] */ + rax = (word64)(WC_L64(rcx, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 224) = (word64)(r9); + /* A[14] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r9 = (word64)(0); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B[14] */ + rax = (word64)(WC_L64(rcx, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 232) = (word64)(r10); + /* A[15] * B[15] */ + rax = (word64)(WC_L64(rcx, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 240) = (word64)(r8); + WC_S64(rdi, 248) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r8 = (word64)(WC_L64(rsp, 16)); + r9 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + r8 = (word64)(WC_L64(rsp, 48)); + r9 = (word64)(WC_L64(rsp, 56)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 64)); + rdx = (word64)(WC_L64(rsp, 72)); + r8 = (word64)(WC_L64(rsp, 80)); + r9 = (word64)(WC_L64(rsp, 88)); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rdx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rsp, 96)); + rdx = (word64)(WC_L64(rsp, 104)); + r8 = (word64)(WC_L64(rsp, 112)); + r9 = (word64)(WC_L64(rsp, 120)); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rdx); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); +} + +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_1024_sqr_16(sp_digit* r, const sp_digit* a) +{ + word64 rdi, rsi, rsp, rax, rdx = 0, r9, r8, rcx, r12, r10, r11; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + /* A[0] * A[0] */ + rax = (word64)(WC_L64(rsi, 0)); + WC_X64I_MUL128(rax, rdx, rax, rax); + r9 = (word64)(0); + WC_S64(rsp, 0) = (word64)(rax); + r8 = (word64)(rdx); + /* A[0] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 8) = (word64)(r8); + /* A[0] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[1] * A[1] */ + rax = (word64)(WC_L64(rsi, 8)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rsp, 16) = (word64)(r9); + /* A[0] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[1] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rsp, 24) = (word64)(rcx); + /* A[0] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[1] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[2] * A[2] */ + rax = (word64)(WC_L64(rsi, 16)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rsp, 32) = (word64)(r8); + /* A[0] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 40) = (word64)(r9); + /* A[0] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[3] */ + rax = (word64)(WC_L64(rsi, 24)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 48) = (word64)(rcx); + /* A[0] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rsp, 56) = (word64)(r8); + /* A[0] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[4] */ + rax = (word64)(WC_L64(rsi, 32)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 64) = (word64)(r9); + /* A[0] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 72) = (word64)(rcx); + /* A[0] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[5] */ + rax = (word64)(WC_L64(rsi, 40)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rsp, 80) = (word64)(r8); + /* A[0] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 88) = (word64)(r9); + /* A[0] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[6] */ + rax = (word64)(WC_L64(rsi, 48)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 96) = (word64)(rcx); + /* A[0] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rsp, 104) = (word64)(r8); + /* A[0] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[7] */ + rax = (word64)(WC_L64(rsi, 56)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rsp, 112) = (word64)(r9); + /* A[0] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[1] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[2] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rsp, 120) = (word64)(rcx); + /* A[1] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[2] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[3] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[8] */ + rax = (word64)(WC_L64(rsi, 64)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 128) = (word64)(r8); + /* A[2] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[3] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[4] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rdi, 136) = (word64)(r9); + /* A[3] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[4] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[5] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[9] */ + rax = (word64)(WC_L64(rsi, 72)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rdi, 144) = (word64)(rcx); + /* A[4] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[5] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[6] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 152) = (word64)(r8); + /* A[5] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[6] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[7] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[10] * A[10] */ + rax = (word64)(WC_L64(rsi, 80)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rdi, 160) = (word64)(r9); + /* A[6] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[7] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[8] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[10] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rdi, 168) = (word64)(rcx); + /* A[7] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[8] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[9] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[10] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[11] * A[11] */ + rax = (word64)(WC_L64(rsi, 88)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 176) = (word64)(r8); + /* A[8] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r8 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[9] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[10] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[11] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r9, r10, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r11, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r12, (unsigned long long*)&r8); + WC_S64(rdi, 184) = (word64)(r9); + /* A[9] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r9 = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[10] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[11] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[12] * A[12] */ + rax = (word64)(WC_L64(rsi, 96)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, rcx, r10, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, r11, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r12, (unsigned long long*)&r9); + WC_S64(rdi, 192) = (word64)(rcx); + /* A[10] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + rcx = (word64)(0); + r12 = (word64)(0); + r10 = (word64)(rax); + r11 = (word64)(rdx); + /* A[11] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + /* A[12] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, rdx, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, 0, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r10, r10, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r11, r11, (unsigned long long*)&r11); + cf = _addcarry_u64(cf, r12, r12, (unsigned long long*)&r12); + cf = _addcarry_u64(0, r8, r10, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, r11, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, r12, (unsigned long long*)&rcx); + WC_S64(rdi, 200) = (word64)(r8); + /* A[11] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[12] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[13] * A[13] */ + rax = (word64)(WC_L64(rsi, 104)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 208) = (word64)(r9); + /* A[12] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r9 = (word64)(0); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[13] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + WC_S64(rdi, 216) = (word64)(rcx); + /* A[13] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + rcx = (word64)(0); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + /* A[14] * A[14] */ + rax = (word64)(WC_L64(rsi, 112)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, 0, (unsigned long long*)&rcx); + WC_S64(rdi, 224) = (word64)(r8); + /* A[14] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r8 = (word64)(0); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, rcx, rdx, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + WC_S64(rdi, 232) = (word64)(r9); + /* A[15] * A[15] */ + rax = (word64)(WC_L64(rsi, 120)); + WC_X64I_MUL128(rax, rdx, rax, rax); + cf = _addcarry_u64(0, rcx, rax, (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 240) = (word64)(rcx); + WC_S64(rdi, 248) = (word64)(r8); + rax = (word64)(WC_L64(rsp, 0)); + rdx = (word64)(WC_L64(rsp, 8)); + r10 = (word64)(WC_L64(rsp, 16)); + r11 = (word64)(WC_L64(rsp, 24)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rdx); + WC_S64(rdi, 16) = (word64)(r10); + WC_S64(rdi, 24) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 32)); + rdx = (word64)(WC_L64(rsp, 40)); + r10 = (word64)(WC_L64(rsp, 48)); + r11 = (word64)(WC_L64(rsp, 56)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rdx); + WC_S64(rdi, 48) = (word64)(r10); + WC_S64(rdi, 56) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 64)); + rdx = (word64)(WC_L64(rsp, 72)); + r10 = (word64)(WC_L64(rsp, 80)); + r11 = (word64)(WC_L64(rsp, 88)); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rdx); + WC_S64(rdi, 80) = (word64)(r10); + WC_S64(rdi, 88) = (word64)(r11); + rax = (word64)(WC_L64(rsp, 96)); + rdx = (word64)(WC_L64(rsp, 104)); + r10 = (word64)(WC_L64(rsp, 112)); + r11 = (word64)(WC_L64(rsp, 120)); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rdx); + WC_S64(rdi, 112) = (word64)(r10); + WC_S64(rdi, 120) = (word64)(r11); +} + +#ifdef HAVE_INTEL_AVX2 +/* Multiply a and b into r. (r = a * b) + * + * @param [out] r Result of multiplication. + * @param [in] a First number to multiply. + * @param [in] b Second number to multiply. + */ +WOLFSSL_LOCAL void sp_1024_mul_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + sp_1024_mul_16(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Square a and put result in r. (r = a * a) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + */ +WOLFSSL_LOCAL void sp_1024_sqr_avx2_16(sp_digit* r, const sp_digit* a) +{ + sp_1024_sqr_16(r, a); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Add b to a into r. (r = a + b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_1024_add_16(sp_digit* r, const sp_digit* a, + const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax, r8; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + /* Add */ + rcx = (word64)(WC_L64(rsi, 0)); + rax = (word64)(0); + cf = _addcarry_u64(0, rcx, WC_L64(rdx, 0), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 8)); + WC_S64(rdi, 0) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 16)); + WC_S64(rdi, 8) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 16), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 24)); + WC_S64(rdi, 16) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 24), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 32)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 32), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 40)); + WC_S64(rdi, 32) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 48)); + WC_S64(rdi, 40) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 48), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 56)); + WC_S64(rdi, 48) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 56), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 64)); + WC_S64(rdi, 56) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 64), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 72)); + WC_S64(rdi, 64) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 80)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 80), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 88)); + WC_S64(rdi, 80) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 88), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 96)); + WC_S64(rdi, 88) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 96), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 104)); + WC_S64(rdi, 96) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + rcx = (word64)(WC_L64(rsi, 112)); + WC_S64(rdi, 104) = (word64)(r8); + cf = _addcarry_u64(cf, rcx, WC_L64(rdx, 112), (unsigned long long*)&rcx); + r8 = (word64)(WC_L64(rsi, 120)); + WC_S64(rdi, 112) = (word64)(rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 120), (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r8); + cf = _addcarry_u64(cf, rax, 0, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Sub b from a into a. (a -= b) + * + * @param [in, out] a A single precision integer and result. + * @param [in] b A single precision integer. + */ +WOLFSSL_LOCAL sp_digit sp_1024_sub_in_place_16(sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rdx, rcx, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rdx = (word64)(WC_L64(rdi, 0)); + cf = _subborrow_u64(0, rdx, WC_L64(rsi, 0), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 8)); + WC_S64(rdi, 0) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 16)); + WC_S64(rdi, 8) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 16), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 24)); + WC_S64(rdi, 16) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 24), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 32)); + WC_S64(rdi, 24) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 32), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 40)); + WC_S64(rdi, 32) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 48)); + WC_S64(rdi, 40) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 48), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 56)); + WC_S64(rdi, 48) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 56), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 64)); + WC_S64(rdi, 56) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 64), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 72)); + WC_S64(rdi, 64) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 80)); + WC_S64(rdi, 72) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 80), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 88)); + WC_S64(rdi, 80) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 88), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 96)); + WC_S64(rdi, 88) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 96), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 104)); + WC_S64(rdi, 96) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + rdx = (word64)(WC_L64(rdi, 112)); + WC_S64(rdi, 104) = (word64)(rcx); + cf = _subborrow_u64(cf, rdx, WC_L64(rsi, 112), (unsigned long long*)&rdx); + rcx = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 112) = (word64)(rdx); + cf = _subborrow_u64(cf, rcx, WC_L64(rsi, 120), (unsigned long long*)&rcx); + WC_S64(rdi, 120) = (word64)(rcx); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL sp_digit sp_1024_cond_sub_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, rsp, r8, r9, rax = 0; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & rcx); + r9 = (word64)(r9 & rcx); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + r8 = (word64)(WC_L64(rsi, 0)); + rdx = (word64)(WC_L64(rsp, 0)); + cf = _subborrow_u64(0, r8, rdx, (unsigned long long*)&r8); + r9 = (word64)(WC_L64(rsi, 8)); + rdx = (word64)(WC_L64(rsp, 8)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 16)); + rdx = (word64)(WC_L64(rsp, 16)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 24)); + rdx = (word64)(WC_L64(rsp, 24)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 32)); + rdx = (word64)(WC_L64(rsp, 32)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 40)); + rdx = (word64)(WC_L64(rsp, 40)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 48)); + rdx = (word64)(WC_L64(rsp, 48)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 56)); + rdx = (word64)(WC_L64(rsp, 56)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 64)); + rdx = (word64)(WC_L64(rsp, 64)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 72)); + rdx = (word64)(WC_L64(rsp, 72)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 80)); + rdx = (word64)(WC_L64(rsp, 80)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 88)); + rdx = (word64)(WC_L64(rsp, 88)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 96)); + rdx = (word64)(WC_L64(rsp, 96)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 104)); + rdx = (word64)(WC_L64(rsp, 104)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rsi, 112)); + rdx = (word64)(WC_L64(rsp, 112)); + cf = _subborrow_u64(cf, r8, rdx, (unsigned long long*)&r8); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rsi, 120)); + rdx = (word64)(WC_L64(rsp, 120)); + cf = _subborrow_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#ifdef HAVE_INTEL_AVX2 +/* Conditionally subtract b from a using the mask m. + * m is -1 to subtract and 0 when not copying. + * + * @param [out] r A single precision number representing condition subtract + * result. + * @param [in] a A single precision number to subtract from. + * @param [in] b A single precision number to subtract. + * @param [in] m Mask value to apply. + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL sp_digit sp_1024_cond_sub_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, sp_digit m) +{ + word64 rdi, rsi, rdx, rcx, r10, r8, r9, rax = 0; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word64)m; + + r10 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + cf = _subborrow_u64(0, r8, r10, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 8)); + r9 = (word64)(WC_L64(rsi, 8)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 0) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 8) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 16) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rsi, 32)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 24) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 32) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 40) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 56)); + r9 = (word64)(WC_L64(rsi, 56)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 48) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 56) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 64) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 72) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 80) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 88) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + r10 = (word64)(WC_L64(rdx, 104)); + r9 = (word64)(WC_L64(rsi, 104)); + r10 = (word64)((word64)_pext_u64(r10, rcx)); + WC_S64(rdi, 96) = (word64)(r8); + cf = _subborrow_u64(cf, r9, r10, (unsigned long long*)&r9); + r8 = (word64)(WC_L64(rdx, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r8 = (word64)((word64)_pext_u64(r8, rcx)); + WC_S64(rdi, 104) = (word64)(r9); + cf = _subborrow_u64(cf, r10, r8, (unsigned long long*)&r10); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)((word64)_pext_u64(r9, rcx)); + WC_S64(rdi, 112) = (word64)(r10); + cf = _subborrow_u64(cf, r8, r9, (unsigned long long*)&r8); + WC_S64(rdi, 120) = (word64)(r8); + cf = _subborrow_u64(cf, rax, rax, (unsigned long long*)&rax); + return (sp_digit)(word64)rax; +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_1024_mul_d_16(sp_digit* r, const sp_digit* a, sp_digit b) +{ + word64 rdi, rsi, rcx, rax, r10, rdx = 0, r8, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rcx = (word64)(word64)b; + + /* A[0] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + r8 = (word64)(rax); + r9 = (word64)(rdx); + WC_S64(rdi, 0) = (word64)(r8); + /* A[1] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 8) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[2] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 16) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[3] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 24) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[4] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[5] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 40) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[6] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[7] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 56) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[8] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[9] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 72) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[10] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[11] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 88) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[12] * B */ + rax = (word64)(rcx); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* A[13] * B */ + rax = (word64)(rcx); + r8 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + cf = _addcarry_u64(0, r9, rax, (unsigned long long*)&r9); + WC_S64(rdi, 104) = (word64)(r9); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, r8, 0, (unsigned long long*)&r8); + /* A[14] * B */ + rax = (word64)(rcx); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + WC_S64(rdi, 112) = (word64)(r10); + cf = _addcarry_u64(cf, r8, rdx, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* A[15] * B */ + rax = (word64)(rcx); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + WC_S64(rdi, 120) = (word64)(r8); + WC_S64(rdi, 128) = (word64)(r9); +} + +#ifdef HAVE_INTEL_AVX2 +/* Mul a by digit b into r. (r = a * b) + * + * @param [out] r A single precision integer. + * @param [in] a A single precision integer. + * @param [in] b A single precision digit. + */ +WOLFSSL_LOCAL void sp_1024_mul_d_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit b) +{ + sp_1024_mul_d_16(r, a, b); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef _WIN64 +/* Divide the double width number (d1|d0) by the dividend. (d1|d0 / div) + * + * @param [in] d1 The high order half of the number to divide. + * @param [in] d0 The low order half of the number to divide. + * @param [in] div The dividend. + * + * @return The result of the division. + */ +WOLFSSL_LOCAL sp_digit div_1024_word_asm_16(sp_digit d1, sp_digit d0, + sp_digit div) +{ + word64 rdi, rsi, rcx, rax, rdx; + + rdi = (word64)(word64)d1; + rsi = (word64)(word64)d0; + rcx = (word64)(word64)div; + + rax = (word64)(rsi); + rdx = (word64)(rdi); + WC_X64I_DIV128(rax, rdx, rdx, rax, rcx); + return (sp_digit)(word64)rax; +} + +#endif /* _WIN64 */ +/* Compare a with b in constant time. + * + * @param [in] a A single precision integer. + * @param [in] b A single precision integer. + * + * @return -ve, 0 or +ve if a is less than, equal to or greater than b + * respectively. + */ +WOLFSSL_LOCAL sp_int64 sp_1024_cmp_16(const sp_digit* a, const sp_digit* b) +{ + word64 rdi, rsi, rcx, rdx, rax, r8, r9, r10; + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + word64 zf7; + word64 zf8; + word64 zf9; + word64 zf10; + word64 zf11; + word64 zf12; + word64 zf13; + word64 zf14; + word64 zf15; + word64 zf16; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + + rcx = (word64)(0); + rdx = (word64)(-1); + rax = (word64)(-1); + r8 = (word64)(1); + r9 = (word64)(WC_L64(rdi, 120)); + r10 = (word64)(WC_L64(rsi, 120)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf1 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf1) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 112)); + r10 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf2 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf2) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 104)); + r10 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf3 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf3) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 96)); + r10 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf4 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf4) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 88)); + r10 = (word64)(WC_L64(rsi, 88)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf5 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf5) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 80)); + r10 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf6 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf6) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 72)); + r10 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf7 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf7) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 64)); + r10 = (word64)(WC_L64(rsi, 64)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf8 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf8) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 56)); + r10 = (word64)(WC_L64(rsi, 56)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf9 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf9) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 48)); + r10 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf10 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf10) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 40)); + r10 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf11 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf11) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 32)); + r10 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf12 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf12) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 24)); + r10 = (word64)(WC_L64(rsi, 24)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf13 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf13) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 16)); + r10 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf14 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf14) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 8)); + r10 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf15 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf15) != (0) ? rcx : rdx; + r9 = (word64)(WC_L64(rdi, 0)); + r10 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(r9 & rdx); + r10 = (word64)(r10 & rdx); + cf = _subborrow_u64(0, r9, r10, (unsigned long long*)&r9); + zf16 = r9; + rax = (!(cf) && ((r9) != (0))) ? r8 : rax; + rax = (cf) ? rdx : rax; + rdx = (zf16) != (0) ? rcx : rdx; + rax = (word64)(rax ^ rdx); + return (sp_int64)(word64)rax; +} + +/* Conditionally copy a into r using the mask m. + * m is -1 to copy and 0 when not. + * + * @param [out] r A single precision number to copy over. + * @param [in] a A single precision number to copy. + * @param [in] m Mask value to apply. + */ +WOLFSSL_LOCAL void sp_1024_cond_copy_16(sp_digit* r, const sp_digit* a, + sp_digit m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(word64)m; + + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)(WC_L64(rdi, 16)); + r9 = (word64)(WC_L64(rdi, 24)); + rax = (word64)(rax ^ WC_L64(rsi, 0)); + rcx = (word64)(rcx ^ WC_L64(rsi, 8)); + r8 = (word64)(r8 ^ WC_L64(rsi, 16)); + r9 = (word64)(r9 ^ WC_L64(rsi, 24)); + rax = (word64)(rax & rdx); + rcx = (word64)(rcx & rdx); + r8 = (word64)(r8 & rdx); + r9 = (word64)(r9 & rdx); + WC_S64(rdi, 0) = (word64)(WC_L64(rdi, 0) ^ rax); + WC_S64(rdi, 8) = (word64)(WC_L64(rdi, 8) ^ rcx); + WC_S64(rdi, 16) = (word64)(WC_L64(rdi, 16) ^ r8); + WC_S64(rdi, 24) = (word64)(WC_L64(rdi, 24) ^ r9); + rax = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + r8 = (word64)(WC_L64(rdi, 48)); + r9 = (word64)(WC_L64(rdi, 56)); + rax = (word64)(rax ^ WC_L64(rsi, 32)); + rcx = (word64)(rcx ^ WC_L64(rsi, 40)); + r8 = (word64)(r8 ^ WC_L64(rsi, 48)); + r9 = (word64)(r9 ^ WC_L64(rsi, 56)); + rax = (word64)(rax & rdx); + rcx = (word64)(rcx & rdx); + r8 = (word64)(r8 & rdx); + r9 = (word64)(r9 & rdx); + WC_S64(rdi, 32) = (word64)(WC_L64(rdi, 32) ^ rax); + WC_S64(rdi, 40) = (word64)(WC_L64(rdi, 40) ^ rcx); + WC_S64(rdi, 48) = (word64)(WC_L64(rdi, 48) ^ r8); + WC_S64(rdi, 56) = (word64)(WC_L64(rdi, 56) ^ r9); + rax = (word64)(WC_L64(rdi, 64)); + rcx = (word64)(WC_L64(rdi, 72)); + r8 = (word64)(WC_L64(rdi, 80)); + r9 = (word64)(WC_L64(rdi, 88)); + rax = (word64)(rax ^ WC_L64(rsi, 64)); + rcx = (word64)(rcx ^ WC_L64(rsi, 72)); + r8 = (word64)(r8 ^ WC_L64(rsi, 80)); + r9 = (word64)(r9 ^ WC_L64(rsi, 88)); + rax = (word64)(rax & rdx); + rcx = (word64)(rcx & rdx); + r8 = (word64)(r8 & rdx); + r9 = (word64)(r9 & rdx); + WC_S64(rdi, 64) = (word64)(WC_L64(rdi, 64) ^ rax); + WC_S64(rdi, 72) = (word64)(WC_L64(rdi, 72) ^ rcx); + WC_S64(rdi, 80) = (word64)(WC_L64(rdi, 80) ^ r8); + WC_S64(rdi, 88) = (word64)(WC_L64(rdi, 88) ^ r9); + rax = (word64)(WC_L64(rdi, 96)); + rcx = (word64)(WC_L64(rdi, 104)); + r8 = (word64)(WC_L64(rdi, 112)); + r9 = (word64)(WC_L64(rdi, 120)); + rax = (word64)(rax ^ WC_L64(rsi, 96)); + rcx = (word64)(rcx ^ WC_L64(rsi, 104)); + r8 = (word64)(r8 ^ WC_L64(rsi, 112)); + r9 = (word64)(r9 ^ WC_L64(rsi, 120)); + rax = (word64)(rax & rdx); + rcx = (word64)(rcx & rdx); + r8 = (word64)(r8 & rdx); + r9 = (word64)(r9 & rdx); + WC_S64(rdi, 96) = (word64)(WC_L64(rdi, 96) ^ rax); + WC_S64(rdi, 104) = (word64)(WC_L64(rdi, 104) ^ rcx); + WC_S64(rdi, 112) = (word64)(WC_L64(rdi, 112) ^ r8); + WC_S64(rdi, 120) = (word64)(WC_L64(rdi, 120) ^ r9); +} + +/* Reduce the number back to 1024 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_1024_mont_reduce_16(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + word64 rdi, rsi, rcx, r15, r8, r13, r14, r11 = 0, rax = 0, r10 = 0, + rdx = 0, r9 = 0, r12 = 0; + unsigned char cf; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)m; + rcx = (word64)(word64)mp; + + r15 = (word64)(0); + /* i = 16 */ + r8 = (word64)(0x10); + r13 = (word64)(WC_L64(rdi, 0)); + r14 = (word64)(WC_L64(rdi, 8)); +L_1024_mont_reduce_16_loop: + /* mu = a[i] * mp */ + r11 = (word64)(r13); + r11 = (word64)(r11 * rcx); + /* a[i+0] += m[0] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 0)); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + /* a[i+1] += m[1] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 8)); + r13 = (word64)(r14); + cf = _addcarry_u64(0, r13, rax, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r13, r10, (unsigned long long*)&r13); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+2] += m[2] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 16)); + r14 = (word64)(WC_L64(rdi, 16)); + cf = _addcarry_u64(0, r14, rax, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r14, r9, (unsigned long long*)&r14); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+3] += m[3] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 24)); + r12 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 24) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+4] += m[4] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 32)); + r12 = (word64)(WC_L64(rdi, 32)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 32) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+5] += m[5] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 40)); + r12 = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 40) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+6] += m[6] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 48)); + r12 = (word64)(WC_L64(rdi, 48)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 48) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+7] += m[7] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 56)); + r12 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 56) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+8] += m[8] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 64)); + r12 = (word64)(WC_L64(rdi, 64)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 64) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+9] += m[9] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 72)); + r12 = (word64)(WC_L64(rdi, 72)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 72) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+10] += m[10] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 80)); + r12 = (word64)(WC_L64(rdi, 80)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 80) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+11] += m[11] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 88)); + r12 = (word64)(WC_L64(rdi, 88)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 88) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+12] += m[12] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 96)); + r12 = (word64)(WC_L64(rdi, 96)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 96) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+13] += m[13] * mu */ + rax = (word64)(r11); + r9 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 104)); + r12 = (word64)(WC_L64(rdi, 104)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r9, rdx, (unsigned long long*)&r9); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 104) = (word64)(r12); + cf = _addcarry_u64(cf, r9, 0, (unsigned long long*)&r9); + /* a[i+14] += m[14] * mu */ + rax = (word64)(r11); + r10 = (word64)(0); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 112)); + r12 = (word64)(WC_L64(rdi, 112)); + cf = _addcarry_u64(0, r12, rax, (unsigned long long*)&r12); + cf = _addcarry_u64(cf, r10, rdx, (unsigned long long*)&r10); + cf = _addcarry_u64(0, r12, r9, (unsigned long long*)&r12); + WC_S64(rdi, 112) = (word64)(r12); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + /* a[i+15] += m[15] * mu */ + rax = (word64)(r11); + WC_X64I_MUL128(rax, rdx, rax, WC_L64(rsi, 120)); + r12 = (word64)(WC_L64(rdi, 120)); + cf = _addcarry_u64(0, r10, rax, (unsigned long long*)&r10); + cf = _addcarry_u64(cf, rdx, r15, (unsigned long long*)&rdx); + r15 = (word64)(0); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + cf = _addcarry_u64(0, r12, r10, (unsigned long long*)&r12); + WC_S64(rdi, 120) = (word64)(r12); + cf = _addcarry_u64(cf, WC_L64(rdi, 128), rdx, (unsigned long long*)&WC_S64( + rdi, 128)); + cf = _addcarry_u64(cf, r15, 0, (unsigned long long*)&r15); + /* i -= 1 */ + rdi = (word64)(rdi + 8); + r8 = (word64)(r8 - 1); + if ((r8) != (0)) { + goto L_1024_mont_reduce_16_loop; + } + r12 = (word64)(WC_L64(rdi, 120)); + WC_S64(rdi, 0) = (word64)(r13); + cf = _subborrow_u64(0, r12, WC_L64(rsi, 120), (unsigned long long*)&r12); + WC_S64(rdi, 8) = (word64)(r14); + cf = _subborrow_u64(cf, r12, r12, (unsigned long long*)&r12); + r15 = (word64)(0 - r15); + r12 = (word64)(~r12); + r15 = (word64)(r15 | r12); +#ifdef _WIN64 + rdx = (word64)(rsi); + rcx = (word64)(r15); +#else + rcx = (word64)(r15); + rdx = (word64)(rsi); +#endif /* _WIN64 */ + rsi = (word64)(rdi); + rdi = (word64)(rdi); + rdi = (word64)(rdi - 0x80); + (void)sp_1024_cond_sub_16((sp_digit*)(size_t)rdi, (const sp_digit*)( + size_t)rsi, (const sp_digit*)(size_t)rdx, (sp_digit)rcx); +} + +/* Add two Montgomery form numbers (r = a + b % m). + * + * @param [out] r Result of addition. + * @param [in] a First number to add in Montgomery form. + * @param [in] b Second number to add in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_1024_mont_add_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rcx, rsp, rax, r8, r9, r10, r11; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(size_t)m; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + rax = (word64)(WC_L64(rsi, 0)); + r8 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(WC_L64(rsi, 16)); + r10 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + r11 = (word64)(0); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 32)); + r8 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(WC_L64(rsi, 48)); + r10 = (word64)(WC_L64(rsi, 56)); + cf = _addcarry_u64(cf, rax, WC_L64(rdx, 32), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 48), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 56), (unsigned long long*)&r10); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + WC_S64(rdi, 48) = (word64)(r9); + WC_S64(rdi, 56) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 64)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)(WC_L64(rsi, 88)); + cf = _addcarry_u64(cf, rax, WC_L64(rdx, 64), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 80), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 88), (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(r8); + WC_S64(rdi, 80) = (word64)(r9); + WC_S64(rdi, 88) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 96)); + r8 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(WC_L64(rsi, 112)); + r10 = (word64)(WC_L64(rsi, 120)); + cf = _addcarry_u64(cf, rax, WC_L64(rdx, 96), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 112), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 120), (unsigned long long*)&r10); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(r8); + WC_S64(rdi, 112) = (word64)(r9); + WC_S64(rdi, 120) = (word64)(r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _subborrow_u64(0, r10, WC_L64(rcx, 120), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r10, r10, (unsigned long long*)&r10); + r10 = (word64)(~r10); + r11 = (word64)(r11 | r10); + r9 = (word64)(WC_L64(rcx, 0)); + r10 = (word64)(WC_L64(rcx, 8)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 0) = (word64)(r9); + WC_S64(rsp, 8) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 16)); + r10 = (word64)(WC_L64(rcx, 24)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 16) = (word64)(r9); + WC_S64(rsp, 24) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 32)); + r10 = (word64)(WC_L64(rcx, 40)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 32) = (word64)(r9); + WC_S64(rsp, 40) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 48)); + r10 = (word64)(WC_L64(rcx, 56)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 48) = (word64)(r9); + WC_S64(rsp, 56) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 64)); + r10 = (word64)(WC_L64(rcx, 72)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 64) = (word64)(r9); + WC_S64(rsp, 72) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 80)); + r10 = (word64)(WC_L64(rcx, 88)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 80) = (word64)(r9); + WC_S64(rsp, 88) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 96)); + r10 = (word64)(WC_L64(rcx, 104)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 96) = (word64)(r9); + WC_S64(rsp, 104) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 112)); + r10 = (word64)(WC_L64(rcx, 120)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 112) = (word64)(r9); + WC_S64(rsp, 120) = (word64)(r10); + rax = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + cf = _subborrow_u64(0, rax, WC_L64(rsp, 0), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 8), (unsigned long long*)&r8); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 16)); + r8 = (word64)(WC_L64(rdi, 24)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 16), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 24), (unsigned long long*)&r8); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 32)); + r8 = (word64)(WC_L64(rdi, 40)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 32), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 40), (unsigned long long*)&r8); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 48)); + r8 = (word64)(WC_L64(rdi, 56)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 48), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 56), (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 64)); + r8 = (word64)(WC_L64(rdi, 72)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 64), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 72), (unsigned long long*)&r8); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 80)); + r8 = (word64)(WC_L64(rdi, 88)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 80), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 88), (unsigned long long*)&r8); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 96)); + r8 = (word64)(WC_L64(rdi, 104)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 96), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 104), (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 112)); + r8 = (word64)(WC_L64(rdi, 120)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 112), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rsp, 120), (unsigned long long*)&r8); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(r8); +} + +/* Double a Montgomery form number (r = a + a % m). + * + * @param [out] r Result of addition. + * @param [in] a Number to double in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_1024_mont_dbl_16(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rsp, rax, rcx, r8, r9, r10; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + r10 = (word64)(0); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 16), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 24), (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 48), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 56), (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(WC_L64(rsi, 72)); + r8 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(WC_L64(rsi, 88)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 80), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 88), (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(WC_L64(rsi, 104)); + r8 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(WC_L64(rsi, 120)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 112), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 120), (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(0, r9, WC_L64(rdx, 120), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)(~r9); + r10 = (word64)(r10 | r9); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + cf = _subborrow_u64(0, rax, WC_L64(rsp, 0), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 8), (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 16)); + rcx = (word64)(WC_L64(rdi, 24)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 16), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 24), (unsigned long long*)&rcx); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 32), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 40), (unsigned long long*)&rcx); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 48)); + rcx = (word64)(WC_L64(rdi, 56)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 48), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 56), (unsigned long long*)&rcx); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 64)); + rcx = (word64)(WC_L64(rdi, 72)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 64), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 72), (unsigned long long*)&rcx); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 80)); + rcx = (word64)(WC_L64(rdi, 88)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 80), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 88), (unsigned long long*)&rcx); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 96)); + rcx = (word64)(WC_L64(rdi, 104)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 96), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 104), (unsigned long long*)&rcx); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 112)); + rcx = (word64)(WC_L64(rdi, 120)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 112), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 120), (unsigned long long*)&rcx); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(rcx); +} + +/* Triple a Montgomery form number (r = a + a + a % m). + * + * @param [out] r Result of addition. + * @param [in] a Number to double in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_1024_mont_tpl_16(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rsp, rax, rcx, r8, r9, r10; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + r10 = (word64)(0); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 16), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 24), (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 48), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 56), (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(WC_L64(rsi, 72)); + r8 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(WC_L64(rsi, 88)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 80), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 88), (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(WC_L64(rsi, 104)); + r8 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(WC_L64(rsi, 120)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 112), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 120), (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(0, r9, WC_L64(rdx, 120), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)(~r9); + r10 = (word64)(r10 | r9); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + cf = _subborrow_u64(0, rax, WC_L64(rsp, 0), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 8), (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 16)); + rcx = (word64)(WC_L64(rdi, 24)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 16), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 24), (unsigned long long*)&rcx); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 32), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 40), (unsigned long long*)&rcx); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 48)); + rcx = (word64)(WC_L64(rdi, 56)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 48), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 56), (unsigned long long*)&rcx); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 64)); + rcx = (word64)(WC_L64(rdi, 72)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 64), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 72), (unsigned long long*)&rcx); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 80)); + rcx = (word64)(WC_L64(rdi, 88)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 80), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 88), (unsigned long long*)&rcx); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 96)); + rcx = (word64)(WC_L64(rdi, 104)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 96), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 104), (unsigned long long*)&rcx); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 112)); + rcx = (word64)(WC_L64(rdi, 120)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 112), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 120), (unsigned long long*)&rcx); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)(WC_L64(rdi, 16)); + r9 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + r10 = (word64)(0); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 16), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 24), (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + r8 = (word64)(WC_L64(rdi, 48)); + r9 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 48), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 56), (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rdi, 64)); + rcx = (word64)(WC_L64(rdi, 72)); + r8 = (word64)(WC_L64(rdi, 80)); + r9 = (word64)(WC_L64(rdi, 88)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 80), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 88), (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rdi, 96)); + rcx = (word64)(WC_L64(rdi, 104)); + r8 = (word64)(WC_L64(rdi, 112)); + r9 = (word64)(WC_L64(rdi, 120)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 112), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 120), (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(0, r9, WC_L64(rdx, 120), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)(~r9); + r10 = (word64)(r10 | r9); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 0) = (word64)(r8); + WC_S64(rsp, 8) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 16) = (word64)(r8); + WC_S64(rsp, 24) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 32) = (word64)(r8); + WC_S64(rsp, 40) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 48) = (word64)(r8); + WC_S64(rsp, 56) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 64) = (word64)(r8); + WC_S64(rsp, 72) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 80) = (word64)(r8); + WC_S64(rsp, 88) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 96) = (word64)(r8); + WC_S64(rsp, 104) = (word64)(r9); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & r10); + r9 = (word64)(r9 & r10); + WC_S64(rsp, 112) = (word64)(r8); + WC_S64(rsp, 120) = (word64)(r9); + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + cf = _subborrow_u64(0, rax, WC_L64(rsp, 0), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 8), (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 16)); + rcx = (word64)(WC_L64(rdi, 24)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 16), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 24), (unsigned long long*)&rcx); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 32), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 40), (unsigned long long*)&rcx); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 48)); + rcx = (word64)(WC_L64(rdi, 56)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 48), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 56), (unsigned long long*)&rcx); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 64)); + rcx = (word64)(WC_L64(rdi, 72)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 64), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 72), (unsigned long long*)&rcx); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 80)); + rcx = (word64)(WC_L64(rdi, 88)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 80), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 88), (unsigned long long*)&rcx); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 96)); + rcx = (word64)(WC_L64(rdi, 104)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 96), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 104), (unsigned long long*)&rcx); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 112)); + rcx = (word64)(WC_L64(rdi, 120)); + cf = _subborrow_u64(cf, rax, WC_L64(rsp, 112), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, WC_L64(rsp, 120), (unsigned long long*)&rcx); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(rcx); +} + +/* Subtract two Montgomery form numbers (r = a - b % m). + * + * @param [out] r Result of addition. + * @param [in] a First number to add in Montgomery form. + * @param [in] b Second number to add in Montgomery form. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_1024_mont_sub_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rcx, rsp, rax, r8, r9, r10, r11; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(size_t)m; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + rax = (word64)(WC_L64(rsi, 0)); + r8 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(WC_L64(rsi, 16)); + r10 = (word64)(WC_L64(rsi, 24)); + cf = _subborrow_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + r11 = (word64)(0); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 32)); + r8 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(WC_L64(rsi, 48)); + r10 = (word64)(WC_L64(rsi, 56)); + cf = _subborrow_u64(cf, rax, WC_L64(rdx, 32), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 48), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 56), (unsigned long long*)&r10); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + WC_S64(rdi, 48) = (word64)(r9); + WC_S64(rdi, 56) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 64)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)(WC_L64(rsi, 88)); + cf = _subborrow_u64(cf, rax, WC_L64(rdx, 64), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 80), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 88), (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(r8); + WC_S64(rdi, 80) = (word64)(r9); + WC_S64(rdi, 88) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 96)); + r8 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(WC_L64(rsi, 112)); + r10 = (word64)(WC_L64(rsi, 120)); + cf = _subborrow_u64(cf, rax, WC_L64(rdx, 96), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 112), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 120), (unsigned long long*)&r10); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(r8); + WC_S64(rdi, 112) = (word64)(r9); + WC_S64(rdi, 120) = (word64)(r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + r9 = (word64)(WC_L64(rcx, 0)); + r10 = (word64)(WC_L64(rcx, 8)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 0) = (word64)(r9); + WC_S64(rsp, 8) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 16)); + r10 = (word64)(WC_L64(rcx, 24)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 16) = (word64)(r9); + WC_S64(rsp, 24) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 32)); + r10 = (word64)(WC_L64(rcx, 40)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 32) = (word64)(r9); + WC_S64(rsp, 40) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 48)); + r10 = (word64)(WC_L64(rcx, 56)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 48) = (word64)(r9); + WC_S64(rsp, 56) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 64)); + r10 = (word64)(WC_L64(rcx, 72)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 64) = (word64)(r9); + WC_S64(rsp, 72) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 80)); + r10 = (word64)(WC_L64(rcx, 88)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 80) = (word64)(r9); + WC_S64(rsp, 88) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 96)); + r10 = (word64)(WC_L64(rcx, 104)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 96) = (word64)(r9); + WC_S64(rsp, 104) = (word64)(r10); + r9 = (word64)(WC_L64(rcx, 112)); + r10 = (word64)(WC_L64(rcx, 120)); + r9 = (word64)(r9 & r11); + r10 = (word64)(r10 & r11); + WC_S64(rsp, 112) = (word64)(r9); + WC_S64(rsp, 120) = (word64)(r10); + rax = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + cf = _addcarry_u64(0, rax, WC_L64(rsp, 0), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 8), (unsigned long long*)&r8); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 16)); + r8 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(cf, rax, WC_L64(rsp, 16), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 24), (unsigned long long*)&r8); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 32)); + r8 = (word64)(WC_L64(rdi, 40)); + cf = _addcarry_u64(cf, rax, WC_L64(rsp, 32), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 40), (unsigned long long*)&r8); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 48)); + r8 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(cf, rax, WC_L64(rsp, 48), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 56), (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 64)); + r8 = (word64)(WC_L64(rdi, 72)); + cf = _addcarry_u64(cf, rax, WC_L64(rsp, 64), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 72), (unsigned long long*)&r8); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 80)); + r8 = (word64)(WC_L64(rdi, 88)); + cf = _addcarry_u64(cf, rax, WC_L64(rsp, 80), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 88), (unsigned long long*)&r8); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 96)); + r8 = (word64)(WC_L64(rdi, 104)); + cf = _addcarry_u64(cf, rax, WC_L64(rsp, 96), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 104), (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(r8); + rax = (word64)(WC_L64(rdi, 112)); + r8 = (word64)(WC_L64(rdi, 120)); + cf = _addcarry_u64(cf, rax, WC_L64(rsp, 112), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rsp, 120), (unsigned long long*)&r8); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(r8); +} + +/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus (prime). + */ +WOLFSSL_LOCAL void sp_1024_mont_div2_16(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rsp, r11, r10, rax, r8, rcx; + XALIGNED(32) WC_X64I_SLOT stk[16]; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rsp = (word64)(size_t)stk + 128; + rsp = (word64)(rsp - 128); + r11 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)(0); + rax = (word64)(r11); + r11 = (word64)(r11 & 1); + r11 = (word64)(0 - r11); + r8 = (word64)(WC_L64(rdx, 0)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 8) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 16)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 24) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 32)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 40) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 48)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 56) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 64)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 72) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 80)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 88) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 96)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 104) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 112)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 112) = (word64)(r8); + r8 = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(r8 & r11); + WC_S64(rsp, 120) = (word64)(r8); + cf = _addcarry_u64(0, WC_L64(rsp, 0), rax, (unsigned long long*)&WC_S64(rsp, + 0)); + rax = (word64)(WC_L64(rsi, 8)); + cf = _addcarry_u64(cf, WC_L64(rsp, 8), rax, (unsigned long long*)&WC_S64( + rsp, 8)); + rax = (word64)(WC_L64(rsi, 16)); + cf = _addcarry_u64(cf, WC_L64(rsp, 16), rax, (unsigned long long*)&WC_S64( + rsp, 16)); + rax = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(cf, WC_L64(rsp, 24), rax, (unsigned long long*)&WC_S64( + rsp, 24)); + rax = (word64)(WC_L64(rsi, 32)); + cf = _addcarry_u64(cf, WC_L64(rsp, 32), rax, (unsigned long long*)&WC_S64( + rsp, 32)); + rax = (word64)(WC_L64(rsi, 40)); + cf = _addcarry_u64(cf, WC_L64(rsp, 40), rax, (unsigned long long*)&WC_S64( + rsp, 40)); + rax = (word64)(WC_L64(rsi, 48)); + cf = _addcarry_u64(cf, WC_L64(rsp, 48), rax, (unsigned long long*)&WC_S64( + rsp, 48)); + rax = (word64)(WC_L64(rsi, 56)); + cf = _addcarry_u64(cf, WC_L64(rsp, 56), rax, (unsigned long long*)&WC_S64( + rsp, 56)); + rax = (word64)(WC_L64(rsi, 64)); + cf = _addcarry_u64(cf, WC_L64(rsp, 64), rax, (unsigned long long*)&WC_S64( + rsp, 64)); + rax = (word64)(WC_L64(rsi, 72)); + cf = _addcarry_u64(cf, WC_L64(rsp, 72), rax, (unsigned long long*)&WC_S64( + rsp, 72)); + rax = (word64)(WC_L64(rsi, 80)); + cf = _addcarry_u64(cf, WC_L64(rsp, 80), rax, (unsigned long long*)&WC_S64( + rsp, 80)); + rax = (word64)(WC_L64(rsi, 88)); + cf = _addcarry_u64(cf, WC_L64(rsp, 88), rax, (unsigned long long*)&WC_S64( + rsp, 88)); + rax = (word64)(WC_L64(rsi, 96)); + cf = _addcarry_u64(cf, WC_L64(rsp, 96), rax, (unsigned long long*)&WC_S64( + rsp, 96)); + rax = (word64)(WC_L64(rsi, 104)); + cf = _addcarry_u64(cf, WC_L64(rsp, 104), rax, (unsigned long long*)&WC_S64( + rsp, 104)); + rax = (word64)(WC_L64(rsi, 112)); + cf = _addcarry_u64(cf, WC_L64(rsp, 112), rax, (unsigned long long*)&WC_S64( + rsp, 112)); + rax = (word64)(WC_L64(rsi, 120)); + cf = _addcarry_u64(cf, WC_L64(rsp, 120), rax, (unsigned long long*)&WC_S64( + rsp, 120)); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + rax = (word64)(WC_L64(rsp, 0)); + rcx = (word64)(WC_L64(rsp, 8)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 0) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 16)); + rcx = (word64)((rcx >> 1) | (rax << 63)); + WC_S64(rdi, 8) = (word64)(rcx); + rcx = (word64)(WC_L64(rsp, 24)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 16) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 32)); + rcx = (word64)((rcx >> 1) | (rax << 63)); + WC_S64(rdi, 24) = (word64)(rcx); + rcx = (word64)(WC_L64(rsp, 40)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 32) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 48)); + rcx = (word64)((rcx >> 1) | (rax << 63)); + WC_S64(rdi, 40) = (word64)(rcx); + rcx = (word64)(WC_L64(rsp, 56)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 48) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 64)); + rcx = (word64)((rcx >> 1) | (rax << 63)); + WC_S64(rdi, 56) = (word64)(rcx); + rcx = (word64)(WC_L64(rsp, 72)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 64) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 80)); + rcx = (word64)((rcx >> 1) | (rax << 63)); + WC_S64(rdi, 72) = (word64)(rcx); + rcx = (word64)(WC_L64(rsp, 88)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 80) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 96)); + rcx = (word64)((rcx >> 1) | (rax << 63)); + WC_S64(rdi, 88) = (word64)(rcx); + rcx = (word64)(WC_L64(rsp, 104)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 96) = (word64)(rax); + rax = (word64)(WC_L64(rsp, 112)); + rcx = (word64)((rcx >> 1) | (rax << 63)); + WC_S64(rdi, 104) = (word64)(rcx); + rcx = (word64)(WC_L64(rsp, 120)); + rax = (word64)((rax >> 1) | (rcx << 63)); + WC_S64(rdi, 112) = (word64)(rax); + rcx = (word64)((rcx >> 1) | (r10 << 63)); + WC_S64(rdi, 120) = (word64)(rcx); +} + +#ifdef HAVE_INTEL_AVX2 +/* Reduce the number back to 1024 bits using Montgomery reduction. + * + * @param [in, out] a A single precision number to reduce in place. + * @param [in] m The single precision number representing the modulus. + * @param [in] mp The digit representing the negative inverse of + * m mod 2^n. + */ +WOLFSSL_LOCAL void sp_1024_mont_reduce_avx2_16(sp_digit* a, const sp_digit* m, + sp_digit mp) +{ + sp_1024_mont_reduce_16(a, m, mp); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Add two Montgomery form numbers (r = a + b % m). + * + * @param [out] r Result of addition. + * @param [in] a First number to add in Montgomery form. + * @param [in] b Second number to add in Montgomery form. + * @param [in] m Modulus (prime). + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_1024_mont_add_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rcx, rax, r8, r9, r10, r11; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(size_t)m; + + rax = (word64)(WC_L64(rsi, 0)); + r8 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(WC_L64(rsi, 16)); + r10 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + r11 = (word64)(0); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 32)); + r8 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(WC_L64(rsi, 48)); + r10 = (word64)(WC_L64(rsi, 56)); + cf = _addcarry_u64(cf, rax, WC_L64(rdx, 32), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 48), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 56), (unsigned long long*)&r10); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + WC_S64(rdi, 48) = (word64)(r9); + WC_S64(rdi, 56) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 64)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)(WC_L64(rsi, 88)); + cf = _addcarry_u64(cf, rax, WC_L64(rdx, 64), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 80), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 88), (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(r8); + WC_S64(rdi, 80) = (word64)(r9); + WC_S64(rdi, 88) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 96)); + r8 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(WC_L64(rsi, 112)); + r10 = (word64)(WC_L64(rsi, 120)); + cf = _addcarry_u64(cf, rax, WC_L64(rdx, 96), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rdx, 112), (unsigned long long*)&r9); + cf = _addcarry_u64(cf, r10, WC_L64(rdx, 120), (unsigned long long*)&r10); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(r8); + WC_S64(rdi, 112) = (word64)(r9); + WC_S64(rdi, 120) = (word64)(r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + cf = _subborrow_u64(0, r10, WC_L64(rcx, 120), (unsigned long long*)&r10); + cf = _subborrow_u64(cf, r10, r10, (unsigned long long*)&r10); + r10 = (word64)(~r10); + r11 = (word64)(r11 | r10); + r9 = (word64)(WC_L64(rcx, 0)); + r10 = (word64)(WC_L64(rcx, 8)); + rax = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _subborrow_u64(0, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 16)); + r10 = (word64)(WC_L64(rcx, 24)); + rax = (word64)(WC_L64(rdi, 16)); + r8 = (word64)(WC_L64(rdi, 24)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 32)); + r10 = (word64)(WC_L64(rcx, 40)); + rax = (word64)(WC_L64(rdi, 32)); + r8 = (word64)(WC_L64(rdi, 40)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 48)); + r10 = (word64)(WC_L64(rcx, 56)); + rax = (word64)(WC_L64(rdi, 48)); + r8 = (word64)(WC_L64(rdi, 56)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 64)); + r10 = (word64)(WC_L64(rcx, 72)); + rax = (word64)(WC_L64(rdi, 64)); + r8 = (word64)(WC_L64(rdi, 72)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 80)); + r10 = (word64)(WC_L64(rcx, 88)); + rax = (word64)(WC_L64(rdi, 80)); + r8 = (word64)(WC_L64(rdi, 88)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 96)); + r10 = (word64)(WC_L64(rcx, 104)); + rax = (word64)(WC_L64(rdi, 96)); + r8 = (word64)(WC_L64(rdi, 104)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 112)); + r10 = (word64)(WC_L64(rcx, 120)); + rax = (word64)(WC_L64(rdi, 112)); + r8 = (word64)(WC_L64(rdi, 120)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _subborrow_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(r8); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Double a Montgomery form number (r = a + a % m). + * + * @param [out] r Result of addition. + * @param [in] a Number to double in Montgomery form. + * @param [in] m Modulus (prime). + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_1024_mont_dbl_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + r10 = (word64)(0); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 16), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 24), (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 48), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 56), (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(WC_L64(rsi, 72)); + r8 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(WC_L64(rsi, 88)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 80), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 88), (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(WC_L64(rsi, 104)); + r8 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(WC_L64(rsi, 120)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 112), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 120), (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(0, r9, WC_L64(rdx, 120), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)(~r9); + r10 = (word64)(r10 | r9); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(0, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + rax = (word64)(WC_L64(rdi, 16)); + rcx = (word64)(WC_L64(rdi, 24)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + rax = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + rax = (word64)(WC_L64(rdi, 48)); + rcx = (word64)(WC_L64(rdi, 56)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + rax = (word64)(WC_L64(rdi, 64)); + rcx = (word64)(WC_L64(rdi, 72)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + rax = (word64)(WC_L64(rdi, 80)); + rcx = (word64)(WC_L64(rdi, 88)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + rax = (word64)(WC_L64(rdi, 96)); + rcx = (word64)(WC_L64(rdi, 104)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + rax = (word64)(WC_L64(rdi, 112)); + rcx = (word64)(WC_L64(rdi, 120)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(rcx); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Triple a Montgomery form number (r = a + a + a % m). + * + * @param [out] r Result of addition. + * @param [in] a Number to double in Montgomery form. + * @param [in] m Modulus (prime). + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_1024_mont_tpl_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, rax, rcx, r8, r9, r10; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + rax = (word64)(WC_L64(rsi, 0)); + rcx = (word64)(WC_L64(rsi, 8)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + cf = _addcarry_u64(0, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + r10 = (word64)(0); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 16), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 24), (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 32)); + rcx = (word64)(WC_L64(rsi, 40)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 48), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 56), (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 64)); + rcx = (word64)(WC_L64(rsi, 72)); + r8 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(WC_L64(rsi, 88)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 80), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 88), (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rsi, 96)); + rcx = (word64)(WC_L64(rsi, 104)); + r8 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(WC_L64(rsi, 120)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 112), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 120), (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(0, r9, WC_L64(rdx, 120), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)(~r9); + r10 = (word64)(r10 | r9); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(0, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + rax = (word64)(WC_L64(rdi, 16)); + rcx = (word64)(WC_L64(rdi, 24)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + rax = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + rax = (word64)(WC_L64(rdi, 48)); + rcx = (word64)(WC_L64(rdi, 56)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + rax = (word64)(WC_L64(rdi, 64)); + rcx = (word64)(WC_L64(rdi, 72)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + rax = (word64)(WC_L64(rdi, 80)); + rcx = (word64)(WC_L64(rdi, 88)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + rax = (word64)(WC_L64(rdi, 96)); + rcx = (word64)(WC_L64(rdi, 104)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + rax = (word64)(WC_L64(rdi, 112)); + rcx = (word64)(WC_L64(rdi, 120)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(rcx); + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)(WC_L64(rdi, 16)); + r9 = (word64)(WC_L64(rdi, 24)); + cf = _addcarry_u64(0, rax, WC_L64(rsi, 0), (unsigned long long*)&rax); + r10 = (word64)(0); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 8), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 16), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 24), (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + r8 = (word64)(WC_L64(rdi, 48)); + r9 = (word64)(WC_L64(rdi, 56)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 32), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 40), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 48), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 56), (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rdi, 64)); + rcx = (word64)(WC_L64(rdi, 72)); + r8 = (word64)(WC_L64(rdi, 80)); + r9 = (word64)(WC_L64(rdi, 88)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 64), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 72), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 80), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 88), (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rdi, 96)); + rcx = (word64)(WC_L64(rdi, 104)); + r8 = (word64)(WC_L64(rdi, 112)); + r9 = (word64)(WC_L64(rdi, 120)); + cf = _addcarry_u64(cf, rax, WC_L64(rsi, 96), (unsigned long long*)&rax); + cf = _addcarry_u64(cf, rcx, WC_L64(rsi, 104), (unsigned long long*)&rcx); + cf = _addcarry_u64(cf, r8, WC_L64(rsi, 112), (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, WC_L64(rsi, 120), (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _subborrow_u64(cf, r10, 0, (unsigned long long*)&r10); + cf = _subborrow_u64(0, r9, WC_L64(rdx, 120), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r9, r9, (unsigned long long*)&r9); + r9 = (word64)(~r9); + r10 = (word64)(r10 | r9); + r8 = (word64)(WC_L64(rdx, 0)); + r9 = (word64)(WC_L64(rdx, 8)); + rax = (word64)(WC_L64(rdi, 0)); + rcx = (word64)(WC_L64(rdi, 8)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(0, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 16)); + r9 = (word64)(WC_L64(rdx, 24)); + rax = (word64)(WC_L64(rdi, 16)); + rcx = (word64)(WC_L64(rdi, 24)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 32)); + r9 = (word64)(WC_L64(rdx, 40)); + rax = (word64)(WC_L64(rdi, 32)); + rcx = (word64)(WC_L64(rdi, 40)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 48)); + r9 = (word64)(WC_L64(rdx, 56)); + rax = (word64)(WC_L64(rdi, 48)); + rcx = (word64)(WC_L64(rdi, 56)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 64)); + r9 = (word64)(WC_L64(rdx, 72)); + rax = (word64)(WC_L64(rdi, 64)); + rcx = (word64)(WC_L64(rdi, 72)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 80)); + r9 = (word64)(WC_L64(rdx, 88)); + rax = (word64)(WC_L64(rdi, 80)); + rcx = (word64)(WC_L64(rdi, 88)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 96)); + r9 = (word64)(WC_L64(rdx, 104)); + rax = (word64)(WC_L64(rdi, 96)); + rcx = (word64)(WC_L64(rdi, 104)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(rcx); + r8 = (word64)(WC_L64(rdx, 112)); + r9 = (word64)(WC_L64(rdx, 120)); + rax = (word64)(WC_L64(rdi, 112)); + rcx = (word64)(WC_L64(rdi, 120)); + r8 = (word64)((word64)_pext_u64(r8, r10)); + r9 = (word64)((word64)_pext_u64(r9, r10)); + cf = _subborrow_u64(cf, rax, r8, (unsigned long long*)&rax); + cf = _subborrow_u64(cf, rcx, r9, (unsigned long long*)&rcx); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(rcx); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Subtract two Montgomery form numbers (r = a - b % m). + * + * @param [out] r Result of addition. + * @param [in] a First number to add in Montgomery form. + * @param [in] b Second number to add in Montgomery form. + * @param [in] m Modulus (prime). + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_1024_mont_sub_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* b, const sp_digit* m) +{ + word64 rdi, rsi, rdx, rcx, rax, r8, r9, r10, r11; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(size_t)m; + + rax = (word64)(WC_L64(rsi, 0)); + r8 = (word64)(WC_L64(rsi, 8)); + r9 = (word64)(WC_L64(rsi, 16)); + r10 = (word64)(WC_L64(rsi, 24)); + cf = _subborrow_u64(0, rax, WC_L64(rdx, 0), (unsigned long long*)&rax); + r11 = (word64)(0); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 8), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 16), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 24), (unsigned long long*)&r10); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + WC_S64(rdi, 16) = (word64)(r9); + WC_S64(rdi, 24) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 32)); + r8 = (word64)(WC_L64(rsi, 40)); + r9 = (word64)(WC_L64(rsi, 48)); + r10 = (word64)(WC_L64(rsi, 56)); + cf = _subborrow_u64(cf, rax, WC_L64(rdx, 32), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 40), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 48), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 56), (unsigned long long*)&r10); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + WC_S64(rdi, 48) = (word64)(r9); + WC_S64(rdi, 56) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 64)); + r8 = (word64)(WC_L64(rsi, 72)); + r9 = (word64)(WC_L64(rsi, 80)); + r10 = (word64)(WC_L64(rsi, 88)); + cf = _subborrow_u64(cf, rax, WC_L64(rdx, 64), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 72), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 80), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 88), (unsigned long long*)&r10); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(r8); + WC_S64(rdi, 80) = (word64)(r9); + WC_S64(rdi, 88) = (word64)(r10); + rax = (word64)(WC_L64(rsi, 96)); + r8 = (word64)(WC_L64(rsi, 104)); + r9 = (word64)(WC_L64(rsi, 112)); + r10 = (word64)(WC_L64(rsi, 120)); + cf = _subborrow_u64(cf, rax, WC_L64(rdx, 96), (unsigned long long*)&rax); + cf = _subborrow_u64(cf, r8, WC_L64(rdx, 104), (unsigned long long*)&r8); + cf = _subborrow_u64(cf, r9, WC_L64(rdx, 112), (unsigned long long*)&r9); + cf = _subborrow_u64(cf, r10, WC_L64(rdx, 120), (unsigned long long*)&r10); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(r8); + WC_S64(rdi, 112) = (word64)(r9); + WC_S64(rdi, 120) = (word64)(r10); + cf = _subborrow_u64(cf, r11, 0, (unsigned long long*)&r11); + r9 = (word64)(WC_L64(rcx, 0)); + r10 = (word64)(WC_L64(rcx, 8)); + rax = (word64)(WC_L64(rdi, 0)); + r8 = (word64)(WC_L64(rdi, 8)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _addcarry_u64(0, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 16)); + r10 = (word64)(WC_L64(rcx, 24)); + rax = (word64)(WC_L64(rdi, 16)); + r8 = (word64)(WC_L64(rdi, 24)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _addcarry_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 32)); + r10 = (word64)(WC_L64(rcx, 40)); + rax = (word64)(WC_L64(rdi, 32)); + r8 = (word64)(WC_L64(rdi, 40)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _addcarry_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 48)); + r10 = (word64)(WC_L64(rcx, 56)); + rax = (word64)(WC_L64(rdi, 48)); + r8 = (word64)(WC_L64(rdi, 56)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _addcarry_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 64)); + r10 = (word64)(WC_L64(rcx, 72)); + rax = (word64)(WC_L64(rdi, 64)); + r8 = (word64)(WC_L64(rdi, 72)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _addcarry_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 64) = (word64)(rax); + WC_S64(rdi, 72) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 80)); + r10 = (word64)(WC_L64(rcx, 88)); + rax = (word64)(WC_L64(rdi, 80)); + r8 = (word64)(WC_L64(rdi, 88)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _addcarry_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 80) = (word64)(rax); + WC_S64(rdi, 88) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 96)); + r10 = (word64)(WC_L64(rcx, 104)); + rax = (word64)(WC_L64(rdi, 96)); + r8 = (word64)(WC_L64(rdi, 104)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _addcarry_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 96) = (word64)(rax); + WC_S64(rdi, 104) = (word64)(r8); + r9 = (word64)(WC_L64(rcx, 112)); + r10 = (word64)(WC_L64(rcx, 120)); + rax = (word64)(WC_L64(rdi, 112)); + r8 = (word64)(WC_L64(rdi, 120)); + r9 = (word64)((word64)_pext_u64(r9, r11)); + r10 = (word64)((word64)_pext_u64(r10, r11)); + cf = _addcarry_u64(cf, rax, r9, (unsigned long long*)&rax); + cf = _addcarry_u64(cf, r8, r10, (unsigned long long*)&r8); + WC_S64(rdi, 112) = (word64)(rax); + WC_S64(rdi, 120) = (word64)(r8); +} + +#endif /* HAVE_INTEL_AVX2 */ +#ifdef HAVE_INTEL_AVX2 +/* Divide the number by 2 mod the modulus (prime). (r = a / 2 % m) + * + * @param [out] r Result of division by 2. + * @param [in] a Number to divide. + * @param [in] m Modulus (prime). + */ +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL void sp_1024_mont_div2_avx2_16(sp_digit* r, const sp_digit* a, + const sp_digit* m) +{ + word64 rdi, rsi, rdx, r11, r10, r8, rax, rcx, r9; + unsigned char cf; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)m; + + r11 = (word64)(WC_L64(rsi, 0)); + r10 = (word64)(0); + r8 = (word64)(r11); + r11 = (word64)(r11 & 1); + r11 = (word64)(0 - r11); + rax = (word64)(WC_L64(rdx, 0)); + rcx = (word64)(WC_L64(rdx, 8)); + r8 = (word64)(WC_L64(rsi, 0)); + r9 = (word64)(WC_L64(rsi, 8)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(0, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 0) = (word64)(r8); + WC_S64(rdi, 8) = (word64)(r9); + rax = (word64)(WC_L64(rdx, 16)); + rcx = (word64)(WC_L64(rdx, 24)); + r8 = (word64)(WC_L64(rsi, 16)); + r9 = (word64)(WC_L64(rsi, 24)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 16) = (word64)(r8); + WC_S64(rdi, 24) = (word64)(r9); + rax = (word64)(WC_L64(rdx, 32)); + rcx = (word64)(WC_L64(rdx, 40)); + r8 = (word64)(WC_L64(rsi, 32)); + r9 = (word64)(WC_L64(rsi, 40)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 32) = (word64)(r8); + WC_S64(rdi, 40) = (word64)(r9); + rax = (word64)(WC_L64(rdx, 48)); + rcx = (word64)(WC_L64(rdx, 56)); + r8 = (word64)(WC_L64(rsi, 48)); + r9 = (word64)(WC_L64(rsi, 56)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 48) = (word64)(r8); + WC_S64(rdi, 56) = (word64)(r9); + rax = (word64)(WC_L64(rdx, 64)); + rcx = (word64)(WC_L64(rdx, 72)); + r8 = (word64)(WC_L64(rsi, 64)); + r9 = (word64)(WC_L64(rsi, 72)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 64) = (word64)(r8); + WC_S64(rdi, 72) = (word64)(r9); + rax = (word64)(WC_L64(rdx, 80)); + rcx = (word64)(WC_L64(rdx, 88)); + r8 = (word64)(WC_L64(rsi, 80)); + r9 = (word64)(WC_L64(rsi, 88)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 80) = (word64)(r8); + WC_S64(rdi, 88) = (word64)(r9); + rax = (word64)(WC_L64(rdx, 96)); + rcx = (word64)(WC_L64(rdx, 104)); + r8 = (word64)(WC_L64(rsi, 96)); + r9 = (word64)(WC_L64(rsi, 104)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 96) = (word64)(r8); + WC_S64(rdi, 104) = (word64)(r9); + rax = (word64)(WC_L64(rdx, 112)); + rcx = (word64)(WC_L64(rdx, 120)); + r8 = (word64)(WC_L64(rsi, 112)); + r9 = (word64)(WC_L64(rsi, 120)); + rax = (word64)((word64)_pext_u64(rax, r11)); + rcx = (word64)((word64)_pext_u64(rcx, r11)); + cf = _addcarry_u64(cf, r8, rax, (unsigned long long*)&r8); + cf = _addcarry_u64(cf, r9, rcx, (unsigned long long*)&r9); + WC_S64(rdi, 112) = (word64)(r8); + WC_S64(rdi, 120) = (word64)(r9); + cf = _addcarry_u64(cf, r10, 0, (unsigned long long*)&r10); + r8 = (word64)(WC_L64(rdi, 0)); + r9 = (word64)(WC_L64(rdi, 8)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 0) = (word64)(r8); + r8 = (word64)(WC_L64(rdi, 16)); + r9 = (word64)((r9 >> 1) | (r8 << 63)); + WC_S64(rdi, 8) = (word64)(r9); + r9 = (word64)(WC_L64(rdi, 24)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 16) = (word64)(r8); + r8 = (word64)(WC_L64(rdi, 32)); + r9 = (word64)((r9 >> 1) | (r8 << 63)); + WC_S64(rdi, 24) = (word64)(r9); + r9 = (word64)(WC_L64(rdi, 40)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 32) = (word64)(r8); + r8 = (word64)(WC_L64(rdi, 48)); + r9 = (word64)((r9 >> 1) | (r8 << 63)); + WC_S64(rdi, 40) = (word64)(r9); + r9 = (word64)(WC_L64(rdi, 56)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 48) = (word64)(r8); + r8 = (word64)(WC_L64(rdi, 64)); + r9 = (word64)((r9 >> 1) | (r8 << 63)); + WC_S64(rdi, 56) = (word64)(r9); + r9 = (word64)(WC_L64(rdi, 72)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 64) = (word64)(r8); + r8 = (word64)(WC_L64(rdi, 80)); + r9 = (word64)((r9 >> 1) | (r8 << 63)); + WC_S64(rdi, 72) = (word64)(r9); + r9 = (word64)(WC_L64(rdi, 88)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 80) = (word64)(r8); + r8 = (word64)(WC_L64(rdi, 96)); + r9 = (word64)((r9 >> 1) | (r8 << 63)); + WC_S64(rdi, 88) = (word64)(r9); + r9 = (word64)(WC_L64(rdi, 104)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 96) = (word64)(r8); + r8 = (word64)(WC_L64(rdi, 112)); + r9 = (word64)((r9 >> 1) | (r8 << 63)); + WC_S64(rdi, 104) = (word64)(r9); + r9 = (word64)(WC_L64(rdi, 120)); + r8 = (word64)((r8 >> 1) | (r9 << 63)); + WC_S64(rdi, 112) = (word64)(r8); + r9 = (word64)((r9 >> 1) | (r10 << 63)); + WC_S64(rdi, 120) = (word64)(r9); +} + +#endif /* HAVE_INTEL_AVX2 */ +/* Read big endian unsigned byte array into r. + * Uses the bswap instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WOLFSSL_LOCAL void sp_1024_from_bin_bswap(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, r11, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x80); + r11 = (word64)(0); + goto L_1024_from_bin_bswap_64_end; +L_1024_from_bin_bswap_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_L64(r9, 56)); + r8 = (word64)(WC_L64(r9, 48)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_L64(r9, 40)); + r8 = (word64)(WC_L64(r9, 32)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_L64(r9, 24)); + r8 = (word64)(WC_L64(r9, 16)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_L64(r9, 8)); + r8 = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + r8 = (word64)(WC_X64I_BSWAP64(r8)); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_1024_from_bin_bswap_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_1024_from_bin_bswap_64_start; + } + goto L_1024_from_bin_bswap_8_end; +L_1024_from_bin_bswap_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_L64(r9, 0)); + rax = (word64)(WC_X64I_BSWAP64(rax)); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_1024_from_bin_bswap_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_1024_from_bin_bswap_8_start; + } + if ((rcx) == (r11)) { + goto L_1024_from_bin_bswap_hi_end; + } + r8 = (word64)(r11); + rax = (word64)(r11); +L_1024_from_bin_bswap_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_1024_from_bin_bswap_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_1024_from_bin_bswap_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_1024_from_bin_bswap_zero_end; + } +L_1024_from_bin_bswap_zero_start: + WC_S64(rdi, 0) = (word64)(r11); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_1024_from_bin_bswap_zero_start; + } +L_1024_from_bin_bswap_zero_end: + ; + (void)rsi; +} + +#ifndef NO_MOVBE_SUPPORT +/* Read big endian unsigned byte array into r. + * Uses the movbe instruction which is an optional instruction. + * + * @param [out] r A single precision integer. + * @param [in] size Maximum number of bytes to convert + * @param [in] a Byte array. + * @param [in] n Number of bytes in array to read. + */ +WC_X64I_TARGET("movbe") +WOLFSSL_LOCAL void sp_1024_from_bin_movbe(sp_digit* r, int size, const byte* a, + int n) +{ + word64 rdi, rsi, rdx, rcx, r9, r10, rax = 0, r8 = 0; + + rdi = (word64)(size_t)r; + rsi = (word64)(word32)size; + rdx = (word64)(size_t)a; + rcx = (word64)(word32)n; + + r9 = (word64)(rdx); + r10 = (word64)(rdi); + r9 = (word64)(r9 + rcx); + r10 = (word64)(r10 + 0x80); + goto L_1024_from_bin_movbe_64_end; +L_1024_from_bin_movbe_64_start: + r9 = (word64)(r9 - 0x40); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 56))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 48))); + WC_S64(rdi, 0) = (word64)(rax); + WC_S64(rdi, 8) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 40))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 32))); + WC_S64(rdi, 16) = (word64)(rax); + WC_S64(rdi, 24) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 24))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 16))); + WC_S64(rdi, 32) = (word64)(rax); + WC_S64(rdi, 40) = (word64)(r8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 8))); + r8 = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 48) = (word64)(rax); + WC_S64(rdi, 56) = (word64)(r8); + rdi = (word64)(rdi + 0x40); + rcx = (word64)(rcx - 0x40); +L_1024_from_bin_movbe_64_end: + if ((sword64)(rcx) > (sword64)(0x3f)) { + goto L_1024_from_bin_movbe_64_start; + } + goto L_1024_from_bin_movbe_8_end; +L_1024_from_bin_movbe_8_start: + r9 = (word64)(r9 - 8); + rax = (word64)(WC_X64I_BSWAP64(WC_L64(r9, 0))); + WC_S64(rdi, 0) = (word64)(rax); + rdi = (word64)(rdi + 8); + rcx = (word64)(rcx - 8); +L_1024_from_bin_movbe_8_end: + if ((sword64)(rcx) > (sword64)(7)) { + goto L_1024_from_bin_movbe_8_start; + } + if ((rcx) == (0)) { + goto L_1024_from_bin_movbe_hi_end; + } + r8 = (word64)(0); + rax = (word64)(0); +L_1024_from_bin_movbe_hi_start: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r8 = (word64)(r8 << 8); + rdx = (word64)(rdx + 1); + r8 = (word64)(r8 + rax); + rcx = (word64)(rcx - 1); + if ((sword64)(rcx) > (sword64)(0)) { + goto L_1024_from_bin_movbe_hi_start; + } + WC_S64(rdi, 0) = (word64)(r8); + rdi = (word64)(rdi + 8); +L_1024_from_bin_movbe_hi_end: + if ((sword64)(rdi) >= (sword64)(r10)) { + goto L_1024_from_bin_movbe_zero_end; + } +L_1024_from_bin_movbe_zero_start: + WC_S64(rdi, 0) = (word64)(0); + rdi = (word64)(rdi + 8); + if ((sword64)(rdi) < (sword64)(r10)) { + goto L_1024_from_bin_movbe_zero_start; + } +L_1024_from_bin_movbe_zero_end: + ; + (void)rsi; +} + +#endif /* !NO_MOVBE_SUPPORT */ +#endif /* WOLFSSL_SP_1024 */ +#endif /* WOLFSSL_SP_X86_64_ASM */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/wc_falcon_fpr_intrin.c b/wolfcrypt/src/wc_falcon_fpr_intrin.c new file mode 100644 index 00000000000..51f7d6b4162 --- /dev/null +++ b/wolfcrypt/src/wc_falcon_fpr_intrin.c @@ -0,0 +1,512 @@ +/* wc_falcon_fpr_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define _WC_BUILDING_WC_FALCON_FPR_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#if defined(HAVE_FALCON) && defined(WOLFSSL_FALCON_FPR_ASM) +extern WOLFSSL_LOCAL word64 fpr_add(word64 x, word64 y); +extern WOLFSSL_LOCAL word64 fpr_sub(word64 x, word64 y); +extern WOLFSSL_LOCAL word64 fpr_neg(word64 x); +extern WOLFSSL_LOCAL word64 fpr_half(word64 x); +extern WOLFSSL_LOCAL word64 fpr_double(word64 x); +extern WOLFSSL_LOCAL word64 fpr_mul(word64 x, word64 y); +extern WOLFSSL_LOCAL word64 fpr_sqr(word64 x); +extern WOLFSSL_LOCAL word64 fpr_div(word64 x, word64 y); +extern WOLFSSL_LOCAL word64 fpr_inv(word64 x); +extern WOLFSSL_LOCAL word64 fpr_sqrt(word64 x); +extern WOLFSSL_LOCAL word64 fpr_of(sword64 i); +extern WOLFSSL_LOCAL sword64 fpr_rint(word64 x); +extern WOLFSSL_LOCAL sword64 fpr_floor(word64 x); +extern WOLFSSL_LOCAL sword64 fpr_trunc(word64 x); +extern WOLFSSL_LOCAL int fpr_lt(word64 x, word64 y); + +#endif +#if defined(HAVE_FALCON) && defined(WOLFSSL_FALCON_FPR_ASM) +/* Native x86_64 (SSE2 scalar-double) backend for the Falcon fpr + * seam. Generated by wolfssl-scripts: falcon/x86_64/falcon.rb. + * Each routine is bit-exact with the round-to-nearest-even + * IEEE-754 emulation in wolfcrypt/src/wc_falcon_fpr.c on all values + * Falcon exercises (no subnormals, no NaN/Inf). + * + * SysV AMD64 ABI only (operands in RDI/RSI, result in RAX); not + * MSVC/Win64. Requires the default MXCSR (round-to-nearest-even, FP + * exceptions masked, FTZ/DAZ off) -- the standard C runtime state. + * DO NOT EDIT. */ + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_add(word64 x, word64 y) +{ + word64 rdi, rsi, rax; + __m128i x0, x1; + + rsi = (word64)(word64)y; + rdi = (word64)(word64)x; + /* r = x + y */ + x0 = _mm_cvtsi64_si128((long long)rdi); + x1 = _mm_cvtsi64_si128((long long)rsi); + x0 = _mm_castpd_si128(_mm_add_sd(_mm_castsi128_pd(x0), _mm_castsi128_pd( + x1))); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_sub(word64 x, word64 y) +{ + word64 rdi, rsi, rax; + __m128i x0, x1; + + rsi = (word64)(word64)y; + rdi = (word64)(word64)x; + /* r = x - y */ + x0 = _mm_cvtsi64_si128((long long)rdi); + x1 = _mm_cvtsi64_si128((long long)rsi); + x0 = _mm_castpd_si128(_mm_sub_sd(_mm_castsi128_pd(x0), _mm_castsi128_pd( + x1))); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_neg(word64 x) +{ + word64 rdi, rax; + __m128i x0, x1; + + rdi = (word64)(word64)x; + /* r = -x (flip the sign bit) */ + x0 = _mm_cvtsi64_si128((long long)rdi); + rax = (word64)(0x8000000000000000ULL); + x1 = _mm_cvtsi64_si128((long long)rax); + x0 = _mm_xor_si128(x0, x1); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_half(word64 x) +{ + word64 rdi, rax; + __m128i x0, x1; + + rdi = (word64)(word64)x; + /* r = x * 0.5 */ + x0 = _mm_cvtsi64_si128((long long)rdi); + rax = (word64)(0x3fe0000000000000ULL); + x1 = _mm_cvtsi64_si128((long long)rax); + x0 = _mm_castpd_si128(_mm_mul_sd(_mm_castsi128_pd(x0), _mm_castsi128_pd( + x1))); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_double(word64 x) +{ + word64 rdi, rax; + __m128i x0; + + rdi = (word64)(word64)x; + /* r = x + x */ + x0 = _mm_cvtsi64_si128((long long)rdi); + x0 = _mm_castpd_si128(_mm_add_sd(_mm_castsi128_pd(x0), _mm_castsi128_pd( + x0))); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_mul(word64 x, word64 y) +{ + word64 rdi, rsi, rax; + __m128i x0, x1; + + rsi = (word64)(word64)y; + rdi = (word64)(word64)x; + /* r = x * y */ + x0 = _mm_cvtsi64_si128((long long)rdi); + x1 = _mm_cvtsi64_si128((long long)rsi); + x0 = _mm_castpd_si128(_mm_mul_sd(_mm_castsi128_pd(x0), _mm_castsi128_pd( + x1))); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_sqr(word64 x) +{ + word64 rdi, rax; + __m128i x0; + + rdi = (word64)(word64)x; + /* r = x * x */ + x0 = _mm_cvtsi64_si128((long long)rdi); + x0 = _mm_castpd_si128(_mm_mul_sd(_mm_castsi128_pd(x0), _mm_castsi128_pd( + x0))); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_div(word64 x, word64 y) +{ + word64 rdi, rsi, rax; + __m128i x0, x1; + + rsi = (word64)(word64)y; + rdi = (word64)(word64)x; + /* r = x / y */ + x0 = _mm_cvtsi64_si128((long long)rdi); + x1 = _mm_cvtsi64_si128((long long)rsi); + x0 = _mm_castpd_si128(_mm_div_sd(_mm_castsi128_pd(x0), _mm_castsi128_pd( + x1))); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_inv(word64 x) +{ + word64 rax, rdi; + __m128i x0, x1; + + rdi = (word64)(word64)x; + /* r = 1.0 / x */ + rax = (word64)(0x3ff0000000000000ULL); + x1 = _mm_cvtsi64_si128((long long)rax); + x0 = _mm_cvtsi64_si128((long long)rdi); + x1 = _mm_castpd_si128(_mm_div_sd(_mm_castsi128_pd(x1), _mm_castsi128_pd( + x0))); + rax = (word64)((word64)_mm_cvtsi128_si64(x1)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_sqrt(word64 x) +{ + word64 rdi, rax; + __m128i x0; + + rdi = (word64)(word64)x; + /* r = sqrt(x) (operand assumed non-negative) */ + x0 = _mm_cvtsi64_si128((long long)rdi); + x0 = _mm_castpd_si128(_mm_sqrt_sd(_mm_castsi128_pd(x0), _mm_castsi128_pd( + x0))); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL word64 fpr_of(sword64 i) +{ + word64 rdi, rax; + __m128i x0 = _mm_setzero_si128(); + + rdi = (word64)(word64)i; + /* r = (double)i (round-to-nearest-even) */ + x0 = _mm_castpd_si128(_mm_cvtsi64_sd(_mm_castsi128_pd(x0), (long long)rdi)); + rax = (word64)((word64)_mm_cvtsi128_si64(x0)); + return (word64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL sword64 fpr_rint(word64 x) +{ + word64 rdi, rax; + __m128i x0; + + rdi = (word64)(word64)x; + /* return = lrint(x) (round-to-nearest, ties to even) */ + x0 = _mm_cvtsi64_si128((long long)rdi); + rax = (word64)((word64)_mm_cvtsd_si64(_mm_castsi128_pd(x0))); + return (sword64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL sword64 fpr_floor(word64 x) +{ + word64 rdi, rax; + __m128i x0, x1 = _mm_setzero_si128(); + unsigned char cf; + + rdi = (word64)(word64)x; + /* return = floor(x) (round toward -inf), pure SSE2 */ + x0 = _mm_cvtsi64_si128((long long)rdi); + /* t = trunc(x) toward zero */ + rax = (word64)((word64)_mm_cvttsd_si64(_mm_castsi128_pd(x0))); + /* tf = (double)t */ + x1 = _mm_castpd_si128(_mm_cvtsi64_sd(_mm_castsi128_pd(x1), (long long)rax)); + /* when x < tf, subtract 1 from t (negative non-integers only) */ + cf = _subborrow_u64((unsigned char)(!(_mm_cvtsd_f64(_mm_castsi128_pd( + x0)) >= _mm_cvtsd_f64(_mm_castsi128_pd(x1)))), rax, 0, ( + unsigned long long*)&rax); + return (sword64)(word64)rax; + (void)cf; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL sword64 fpr_trunc(word64 x) +{ + word64 rdi, rax; + __m128i x0; + + rdi = (word64)(word64)x; + /* return = trunc(x) (round toward zero) */ + x0 = _mm_cvtsi64_si128((long long)rdi); + rax = (word64)((word64)_mm_cvttsd_si64(_mm_castsi128_pd(x0))); + return (sword64)(word64)rax; +} + +WC_X64I_TARGET("sse2") +WOLFSSL_LOCAL int fpr_lt(word64 x, word64 y) +{ + word64 rdi, rsi, rax; + __m128i x0, x1; + + rsi = (word64)(word64)y; + rdi = (word64)(word64)x; + /* return = (x < y) ? 1 : 0 */ + x0 = _mm_cvtsi64_si128((long long)rdi); + x1 = _mm_cvtsi64_si128((long long)rsi); + /* clear eax (and flags) before the ordered compare */ + rax = (word32)(0); + /* CF is set iff x < y */ + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)((unsigned char)(!( + _mm_cvtsd_f64(_mm_castsi128_pd(x0)) >= _mm_cvtsd_f64(_mm_castsi128_pd( + x1)))))) & 0xff); + return (int)(word32)rax; +} + +#endif /* HAVE_FALCON && WOLFSSL_FALCON_FPR_ASM */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/wc_frodokem_intrin.c b/wolfcrypt/src/wc_frodokem_intrin.c new file mode 100644 index 00000000000..8b85075c202 --- /dev/null +++ b/wolfcrypt/src/wc_frodokem_intrin.c @@ -0,0 +1,1999 @@ +/* wc_frodokem_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define _WC_BUILDING_WC_FRODOKEM_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_HAVE_FRODOKEM +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void frodokem_sa_accum_avx2(word16* out, const word16* s, + const word16* row, int j, int n); +extern WOLFSSL_LOCAL void frodokem_as_accum_avx2(word16* out, const word16* s, + const word16* row, int i, int n); +extern WOLFSSL_LOCAL void frodokem_mul_bs_avx2(word16* out, const word16* b, + const word16* s, int n, int qmask); +extern WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e_avx2(word16* out, + const word16* b, const word16* s, int n, int qmask); +extern WOLFSSL_LOCAL void frodokem_add_avx2(word16* a, const word16* b, + int qmask); +extern WOLFSSL_LOCAL void frodokem_a_rows_reduce_avx2(word16* rows, word32 cnt, + int qmask); +extern WOLFSSL_LOCAL void frodokem_sample_avx2(word16* mat, int cnt, + const word16* cdf, int cdflen); +extern WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_avx2(byte* in, word16* out, + const byte* ks, int i, int cnt, int n, int qmask); +extern WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_aesni(byte* in, word16* out, + const byte* ks, int i, int cnt, int n, int qmask); +#endif +#ifdef HAVE_INTEL_AVX512 +extern WOLFSSL_LOCAL void frodokem_sa_accum_avx512(word16* out, const word16* s, + const word16* row, int j, int n); +extern WOLFSSL_LOCAL void frodokem_as_accum_avx512(word16* out, const word16* s, + const word16* row, int i, int n); +extern WOLFSSL_LOCAL void frodokem_mul_bs_avx512(word16* out, const word16* b, + const word16* s, int n, int qmask); +extern WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e_avx512(word16* out, + const word16* b, const word16* s, int n, int qmask); +extern WOLFSSL_LOCAL void frodokem_add_avx512(word16* a, const word16* b, + int qmask); +extern WOLFSSL_LOCAL void frodokem_sample_avx512(word16* mat, int cnt, + const word16* cdf, int cdflen); +extern WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_avx512(byte* in, word16* out, + const byte* ks, int i, int cnt, int n, int qmask); + +#endif +#endif +#ifdef WOLFSSL_HAVE_FRODOKEM +#ifdef HAVE_INTEL_AVX2 +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void frodokem_sa_accum_avx2(word16* out, const word16* s, + const word16* row, int j, int n) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11, r13, r12 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(); + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)s; + rdx = (word64)(size_t)row; + rcx = (word64)(word32)j; + r8 = (word64)(word32)n; + + rcx = (word64)((word64)(sword64)(sword32)(word32)rcx); + rcx = (word64)(rcx << 1); + rsi = (word64)(rsi + rcx); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + rax = (word64)(rax << 1); + r9 = (word64)(rdx); + r9 = (word64)(r9 + rax); + r10 = (word64)(r9); + r10 = (word64)(r10 + rax); + r11 = (word64)(r10); + r11 = (word64)(r11 + rax); + r13 = (word64)(8); +L_frodokem_sa_accum_avx2_i: + y0 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 0))); + y1 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 2))); + y2 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 4))); + y3 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 6))); + r12 = (word64)(0); +L_frodokem_sa_accum_avx2_k: + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, r12)); + y5 = _mm256_mullo_epi16(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + r12))); + y4 = _mm256_add_epi16(y4, y5); + y5 = _mm256_mullo_epi16(y1, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + r12))); + y4 = _mm256_add_epi16(y4, y5); + y5 = _mm256_mullo_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + r12))); + y4 = _mm256_add_epi16(y4, y5); + y5 = _mm256_mullo_epi16(y3, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + r12))); + y4 = _mm256_add_epi16(y4, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, r12), y4); + r12 = (word64)(r12 + 0x20); + if ((sword64)(r12) < (sword64)(rax)) { + goto L_frodokem_sa_accum_avx2_k; + } + rdi = (word64)(rdi + rax); + rsi = (word64)(rsi + rax); + r13 = (word64)(r13 - 1); + if ((r13) != (0)) { + goto L_frodokem_sa_accum_avx2_i; + } +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void frodokem_as_accum_avx2(word16* out, const word16* s, + const word16* row, int i, int n) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11, r12, r13, r14, r15, rbp, + rbx = 0; + __m128i x9 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(); + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(); + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)s; + rdx = (word64)(size_t)row; + rcx = (word64)(word32)i; + r8 = (word64)(word32)n; + + rcx = (word64)((word64)(sword64)(sword32)(word32)rcx); + rcx = (word64)(rcx << 4); + rdi = (word64)(rdi + rcx); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + rax = (word64)(rax << 1); + r9 = (word64)(rsi); + r9 = (word64)(r9 + rax); + r10 = (word64)(r9); + r10 = (word64)(r10 + rax); + r11 = (word64)(r10); + r11 = (word64)(r11 + rax); + r12 = (word64)(r11); + r12 = (word64)(r12 + rax); + r13 = (word64)(r12); + r13 = (word64)(r13 + rax); + r14 = (word64)(r13); + r14 = (word64)(r14 + rax); + r15 = (word64)(r14); + r15 = (word64)(r15 + rax); + rbp = (word64)(4); + x12 = _mm_cmpeq_epi32(x12, x12); + x12 = _mm_srli_epi32(x12, 16); +L_frodokem_as_accum_avx2_r: + y0 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + rbx = (word64)(0); +L_frodokem_as_accum_avx2_j: + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, rbx)); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + rbx))); + y0 = _mm256_add_epi32(y0, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + rbx))); + y1 = _mm256_add_epi32(y1, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + rbx))); + y2 = _mm256_add_epi32(y2, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + rbx))); + y3 = _mm256_add_epi32(y3, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + rbx))); + y4 = _mm256_add_epi32(y4, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + rbx))); + y5 = _mm256_add_epi32(y5, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + rbx))); + y6 = _mm256_add_epi32(y6, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + rbx))); + y7 = _mm256_add_epi32(y7, y10); + rbx = (word64)(rbx + 0x20); + if ((sword64)(rbx) < (sword64)(rax)) { + goto L_frodokem_as_accum_avx2_j; + } + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y1 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y2 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y3 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y4 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y5 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y6, 1)); + y6 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y7, 1)); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y10 = _mm256_zextsi128_si256(_mm_hadd_epi32(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + x11 = _mm_hadd_epi32(_mm256_castsi256_si128(y2), _mm256_castsi256_si128( + y3)); + x9 = _mm_hadd_epi32(_mm256_castsi256_si128(y10), x11); + y10 = _mm256_zextsi128_si256(_mm_hadd_epi32(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y5))); + x11 = _mm_hadd_epi32(_mm256_castsi256_si128(y6), _mm256_castsi256_si128( + y7)); + y10 = _mm256_zextsi128_si256(_mm_hadd_epi32(_mm256_castsi256_si128(y10), + x11)); + x9 = _mm_and_si128(x9, x12); + y10 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y10), + x12)); + x9 = _mm_packus_epi32(x9, _mm256_castsi256_si128(y10)); + x9 = _mm_add_epi16(x9, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x9); + rdx = (word64)(rdx + rax); + rdi = (word64)(rdi + 0x10); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_frodokem_as_accum_avx2_r; + } +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void frodokem_mul_bs_avx2(word16* out, const word16* b, + const word16* s, int n, int qmask) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11, r12, r13, r14, r15, rbp, + rbx = 0; + __m128i x9 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), x12; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y10 = _mm256_setzero_si256(); + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)b; + rdx = (word64)(size_t)s; + rcx = (word64)(word32)n; + r8 = (word64)(word32)qmask; + + rax = (word64)((word64)(sword64)(sword32)(word32)rcx); + rax = (word64)(rax << 1); + r9 = (word64)(rdx); + r9 = (word64)(r9 + rax); + r10 = (word64)(r9); + r10 = (word64)(r10 + rax); + r11 = (word64)(r10); + r11 = (word64)(r11 + rax); + r12 = (word64)(r11); + r12 = (word64)(r12 + rax); + r13 = (word64)(r12); + r13 = (word64)(r13 + rax); + r14 = (word64)(r13); + r14 = (word64)(r14 + rax); + r15 = (word64)(r14); + r15 = (word64)(r15 + rax); + rbp = (word64)(8); + x12 = _mm_cvtsi32_si128((int)(word32)r8); + x12 = _mm_broadcastd_epi32(x12); +L_frodokem_mul_bs_avx2_r: + y0 = _mm256_setzero_si256(); + y1 = _mm256_setzero_si256(); + y2 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + y4 = _mm256_setzero_si256(); + y5 = _mm256_setzero_si256(); + y6 = _mm256_setzero_si256(); + y7 = _mm256_setzero_si256(); + rbx = (word64)(0); +L_frodokem_mul_bs_avx2_j: + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, rbx)); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + rbx))); + y0 = _mm256_add_epi32(y0, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + rbx))); + y1 = _mm256_add_epi32(y1, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + rbx))); + y2 = _mm256_add_epi32(y2, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + rbx))); + y3 = _mm256_add_epi32(y3, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r12, + rbx))); + y4 = _mm256_add_epi32(y4, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + rbx))); + y5 = _mm256_add_epi32(y5, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + rbx))); + y6 = _mm256_add_epi32(y6, y10); + y10 = _mm256_madd_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + rbx))); + y7 = _mm256_add_epi32(y7, y10); + rbx = (word64)(rbx + 0x20); + if ((sword64)(rbx) < (sword64)(rax)) { + goto L_frodokem_mul_bs_avx2_j; + } + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y1 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y2 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y2), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y3 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y4 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y5 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y5), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y6, 1)); + y6 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y8))); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y7, 1)); + y7 = _mm256_zextsi128_si256(_mm_add_epi32(_mm256_castsi256_si128(y7), + _mm256_castsi256_si128(y8))); + y10 = _mm256_zextsi128_si256(_mm_hadd_epi32(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1))); + x11 = _mm_hadd_epi32(_mm256_castsi256_si128(y2), _mm256_castsi256_si128( + y3)); + x9 = _mm_hadd_epi32(_mm256_castsi256_si128(y10), x11); + y10 = _mm256_zextsi128_si256(_mm_hadd_epi32(_mm256_castsi256_si128(y4), + _mm256_castsi256_si128(y5))); + x11 = _mm_hadd_epi32(_mm256_castsi256_si128(y6), _mm256_castsi256_si128( + y7)); + y10 = _mm256_zextsi128_si256(_mm_hadd_epi32(_mm256_castsi256_si128(y10), + x11)); + x9 = _mm_and_si128(x9, x12); + y10 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y10), + x12)); + x9 = _mm_packus_epi32(x9, _mm256_castsi256_si128(y10)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x9); + rsi = (word64)(rsi + rax); + rdi = (word64)(rdi + 0x10); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_frodokem_mul_bs_avx2_r; + } +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e_avx2(word16* out, const word16* b, + const word16* s, int n, int qmask) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11, r12, r13, r14, r15, rbx; + __m256i y0, y1, y2, y3, y4 = _mm256_setzero_si256(), + y5 = _mm256_setzero_si256(), y6 = _mm256_setzero_si256(), y7; + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)b; + rdx = (word64)(size_t)s; + rcx = (word64)(word32)n; + r8 = (word64)(word32)qmask; + + rax = (word64)((word64)(sword64)(sword32)(word32)rcx); + rax = (word64)(rax << 1); + r9 = (word64)(rdx); + r9 = (word64)(r9 + rax); + r10 = (word64)(r9); + r10 = (word64)(r10 + rax); + r11 = (word64)(r10); + r11 = (word64)(r11 + rax); + r12 = (word64)(r11); + r12 = (word64)(r12 + rax); + r13 = (word64)(r12); + r13 = (word64)(r13 + rax); + r14 = (word64)(r13); + r14 = (word64)(r14 + rax); + r15 = (word64)(r14); + r15 = (word64)(r15 + rax); + y7 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r8)); + y7 = _mm256_broadcastw_epi16(_mm256_castsi256_si128(y7)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + rbx = (word64)(0); +L_frodokem_mul_add_sb_plus_e_avx2_j: + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 0))); + y5 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + rbx))); + y6 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r9, + rbx))); + y5 = _mm256_inserti128_si256(y5, _mm256_castsi256_si128(y6), 1); + y6 = _mm256_mullo_epi16(y5, y4); + y0 = _mm256_add_epi16(y0, y6); + y5 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r10, + rbx))); + y6 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r11, + rbx))); + y5 = _mm256_inserti128_si256(y5, _mm256_castsi256_si128(y6), 1); + y6 = _mm256_mullo_epi16(y5, y4); + y1 = _mm256_add_epi16(y1, y6); + y5 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r12, + rbx))); + y6 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + y5 = _mm256_inserti128_si256(y5, _mm256_castsi256_si128(y6), 1); + y6 = _mm256_mullo_epi16(y5, y4); + y2 = _mm256_add_epi16(y2, y6); + y5 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r14, + rbx))); + y6 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r15, + rbx))); + y5 = _mm256_inserti128_si256(y5, _mm256_castsi256_si128(y6), 1); + y6 = _mm256_mullo_epi16(y5, y4); + y3 = _mm256_add_epi16(y3, y6); + rsi = (word64)(rsi + 0x10); + rbx = (word64)(rbx + 2); + if ((sword64)(rbx) < (sword64)(rax)) { + goto L_frodokem_mul_add_sb_plus_e_avx2_j; + } + y0 = _mm256_and_si256(y0, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + y1 = _mm256_and_si256(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + y2 = _mm256_and_si256(y2, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + y3 = _mm256_and_si256(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void frodokem_add_avx2(word16* a, const word16* b, int qmask) +{ + word64 rdi, rsi, rdx; + __m256i y0, y1, y2, y3, y4; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + rdx = (word64)(word32)qmask; + + y0 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + y0 = _mm256_broadcastw_epi16(_mm256_castsi256_si128(y0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_add_epi16(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y1 = _mm256_and_si256(y1, y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_add_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + y2 = _mm256_and_si256(y2, y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_add_epi16(y3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y3 = _mm256_and_si256(y3, y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y3); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_add_epi16(y4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + y4 = _mm256_and_si256(y4, y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void frodokem_a_rows_reduce_avx2(word16* rows, word32 cnt, + int qmask) +{ + word64 rdi, rsi, rdx, rax, rcx = 0; + __m256i y0, y1 = _mm256_setzero_si256(); + + rdi = (word64)(size_t)rows; + rsi = (word64)(word32)cnt; + rdx = (word64)(word32)qmask; + + y0 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rdx)); + y0 = _mm256_broadcastw_epi16(_mm256_castsi256_si128(y0)); + rax = (word32)((word32)rsi); +L_frodokem_a_rows_reduce_avx2_blk: + if ((sword64)(rax) < (sword64)(0x10)) { + goto L_frodokem_a_rows_reduce_avx2_tail; + } + y1 = _mm256_and_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + 0x20); + rax = (word64)(rax - 0x10); + goto L_frodokem_a_rows_reduce_avx2_blk; +L_frodokem_a_rows_reduce_avx2_tail: + if ((sword64)(rax) <= (sword64)(0)) { + goto L_frodokem_a_rows_reduce_avx2_done; + } +L_frodokem_a_rows_reduce_avx2_word: + rcx = (word32)((word32)WC_L16(rdi, 0)); + rcx = (word32)((word32)rcx & (word32)rdx); + WC_S16(rdi, 0) = (word16)((word16)rcx); + rdi = (word64)(rdi + 2); + rax = (word64)(rax - 1); + if ((sword64)(rax) > (sword64)(0)) { + goto L_frodokem_a_rows_reduce_avx2_word; + } +L_frodokem_a_rows_reduce_avx2_done: + ; +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void frodokem_sample_avx2(word16* mat, int cnt, const word16* cdf, + int cdflen) +{ + word64 rdi, rsi, rdx, rcx, rax, r8, r9 = 0, r10 = 0; + __m256i y0 = _mm256_setzero_si256(), y1, y2 = _mm256_setzero_si256(), + y3 = _mm256_setzero_si256(), y4 = _mm256_setzero_si256(), + y5 = _mm256_setzero_si256(), y6 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(); + + rdi = (word64)(size_t)mat; + rsi = (word64)(word32)cnt; + rdx = (word64)(size_t)cdf; + rcx = (word64)(word32)cdflen; + + rax = (word64)((word64)(sword64)(sword32)(word32)rsi); + rax = (word64)(rax << 1); + y0 = _mm256_cmpeq_epi16(y0, y0); + y0 = _mm256_srli_epi16(y0, 15); + y1 = _mm256_setzero_si256(); + r8 = (word64)(0); +L_frodokem_sample_avx2_blk: + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, r8)); + y3 = _mm256_srli_epi16(y2, 1); + y4 = _mm256_and_si256(y2, y0); + y5 = _mm256_setzero_si256(); + r9 = (word64)(rdx); + r10 = (word32)((word32)rcx); +L_frodokem_sample_avx2_cdf: + y6 = _mm256_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + y6 = _mm256_sub_epi16(y6, y3); + y6 = _mm256_srli_epi16(y6, 15); + y5 = _mm256_add_epi16(y5, y6); + r9 = (word64)(r9 + 2); + r10 = (word64)(r10 - 1); + if ((r10) != (0)) { + goto L_frodokem_sample_avx2_cdf; + } + y7 = _mm256_sub_epi16(y1, y4); + y5 = _mm256_xor_si256(y5, y7); + y5 = _mm256_add_epi16(y5, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, r8), y5); + r8 = (word64)(r8 + 0x20); + if ((sword64)(r8) < (sword64)(rax)) { + goto L_frodokem_sample_avx2_blk; + } +} + +WC_X64I_TARGET("aes,vaes,avx2") +WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_avx2(byte* in, word16* out, + const byte* ks, int i, int cnt, int n, int qmask) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11, r12, r14 = 0, r13 = 0, + r15 = 0; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, + y11 = _mm256_setzero_si256(), y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(), y14 = _mm256_setzero_si256(), y15; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ks; + rcx = (word64)(word32)i; + r8 = (word64)(word32)cnt; + r9 = (word64)(word32)n; + rax = (word64)(word32)qmask; + + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 0))); + y1 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 16))); + y2 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 32))); + y3 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 48))); + y4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 64))); + y5 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 80))); + y6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 96))); + y7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 112))); + y8 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 128))); + y9 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 144))); + y10 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + y15 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)rax)); + y15 = _mm256_broadcastw_epi16(_mm256_castsi256_si128(y15)); + r10 = (word64)((word64)(sword64)(sword32)(word32)r9); + r10 = (word64)(r10 >> 3); + r11 = (word64)((word64)(sword64)(sword32)(word32)r8); + rcx = (word64)((word64)(sword64)(sword32)(word32)rcx); + r12 = (word64)(0); +L_frodokem_gen_a_rows_aes_avx2_row: + r14 = (word64)(rcx); + r14 = (word64)(r14 + r12); + r13 = (word64)(0); +L_frodokem_gen_a_rows_aes_avx2_blk: + r15 = (word64)(r13 + 8); + if ((sword64)(r15) > (sword64)(r10)) { + goto L_frodokem_gen_a_rows_aes_avx2_tail; + } + y11 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r14)); + r14 = (word64)(r14 + 0x80000); + y0 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r14)); + r14 = (word64)(r14 + 0x80000); + y11 = _mm256_inserti128_si256(y11, _mm256_castsi256_si128(y0), 1); + y12 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r14)); + r14 = (word64)(r14 + 0x80000); + y0 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r14)); + r14 = (word64)(r14 + 0x80000); + y12 = _mm256_inserti128_si256(y12, _mm256_castsi256_si128(y0), 1); + y13 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r14)); + r14 = (word64)(r14 + 0x80000); + y0 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r14)); + r14 = (word64)(r14 + 0x80000); + y13 = _mm256_inserti128_si256(y13, _mm256_castsi256_si128(y0), 1); + y14 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r14)); + r14 = (word64)(r14 + 0x80000); + y0 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r14)); + r14 = (word64)(r14 + 0x80000); + y14 = _mm256_inserti128_si256(y14, _mm256_castsi256_si128(y0), 1); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 0))); + y11 = _mm256_xor_si256(y11, y0); + y12 = _mm256_xor_si256(y12, y0); + y13 = _mm256_xor_si256(y13, y0); + y14 = _mm256_xor_si256(y14, y0); + y11 = _mm256_aesenc_epi128(y11, y1); + y12 = _mm256_aesenc_epi128(y12, y1); + y13 = _mm256_aesenc_epi128(y13, y1); + y14 = _mm256_aesenc_epi128(y14, y1); + y11 = _mm256_aesenc_epi128(y11, y2); + y12 = _mm256_aesenc_epi128(y12, y2); + y13 = _mm256_aesenc_epi128(y13, y2); + y14 = _mm256_aesenc_epi128(y14, y2); + y11 = _mm256_aesenc_epi128(y11, y3); + y12 = _mm256_aesenc_epi128(y12, y3); + y13 = _mm256_aesenc_epi128(y13, y3); + y14 = _mm256_aesenc_epi128(y14, y3); + y11 = _mm256_aesenc_epi128(y11, y4); + y12 = _mm256_aesenc_epi128(y12, y4); + y13 = _mm256_aesenc_epi128(y13, y4); + y14 = _mm256_aesenc_epi128(y14, y4); + y11 = _mm256_aesenc_epi128(y11, y5); + y12 = _mm256_aesenc_epi128(y12, y5); + y13 = _mm256_aesenc_epi128(y13, y5); + y14 = _mm256_aesenc_epi128(y14, y5); + y11 = _mm256_aesenc_epi128(y11, y6); + y12 = _mm256_aesenc_epi128(y12, y6); + y13 = _mm256_aesenc_epi128(y13, y6); + y14 = _mm256_aesenc_epi128(y14, y6); + y11 = _mm256_aesenc_epi128(y11, y7); + y12 = _mm256_aesenc_epi128(y12, y7); + y13 = _mm256_aesenc_epi128(y13, y7); + y14 = _mm256_aesenc_epi128(y14, y7); + y11 = _mm256_aesenc_epi128(y11, y8); + y12 = _mm256_aesenc_epi128(y12, y8); + y13 = _mm256_aesenc_epi128(y13, y8); + y14 = _mm256_aesenc_epi128(y14, y8); + y11 = _mm256_aesenc_epi128(y11, y9); + y12 = _mm256_aesenc_epi128(y12, y9); + y13 = _mm256_aesenc_epi128(y13, y9); + y14 = _mm256_aesenc_epi128(y14, y9); + y11 = _mm256_aesenclast_epi128(y11, y10); + y12 = _mm256_aesenclast_epi128(y12, y10); + y13 = _mm256_aesenclast_epi128(y13, y10); + y14 = _mm256_aesenclast_epi128(y14, y10); + y11 = _mm256_and_si256(y11, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y11); + y12 = _mm256_and_si256(y12, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y12); + y13 = _mm256_and_si256(y13, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y13); + y14 = _mm256_and_si256(y14, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y14); + rsi = (word64)(rsi + 0x80); + r13 = (word64)(r13 + 8); + goto L_frodokem_gen_a_rows_aes_avx2_blk; +L_frodokem_gen_a_rows_aes_avx2_tail: + if ((sword64)(r13) >= (sword64)(r10)) { + goto L_frodokem_gen_a_rows_aes_avx2_next; + } + y11 = _mm256_zextsi128_si256(_mm_cvtsi32_si128((int)(word32)r14)); + r14 = (word64)(r14 + 0x80000); + y11 = _mm256_zextsi128_si256(_mm_xor_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y0))); + y11 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y1))); + y11 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y2))); + y11 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y3))); + y11 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y4))); + y11 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y5))); + y11 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y6))); + y11 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y7))); + y11 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y8))); + y11 = _mm256_zextsi128_si256(_mm_aesenc_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y9))); + y11 = _mm256_zextsi128_si256(_mm_aesenclast_si128(_mm256_castsi256_si128( + y11), _mm256_castsi256_si128(y10))); + y11 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y11), + _mm256_castsi256_si128(y15))); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y11)); + rsi = (word64)(rsi + 0x10); + r13 = (word64)(r13 + 1); + goto L_frodokem_gen_a_rows_aes_avx2_tail; +L_frodokem_gen_a_rows_aes_avx2_next: + r12 = (word64)(r12 + 1); + if ((sword64)(r12) < (sword64)(r11)) { + goto L_frodokem_gen_a_rows_aes_avx2_row; + } + (void)rdi; +} + +WC_X64I_TARGET("aes,avx2") +WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_aesni(byte* in, word16* out, + const byte* ks, int i, int cnt, int n, int qmask) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11, r12, r14 = 0, r13 = 0, + r15 = 0; + __m128i x0 = _mm_setzero_si128(), x1 = _mm_setzero_si128(), + x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), x8; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ks; + rcx = (word64)(word32)i; + r8 = (word64)(word32)cnt; + r9 = (word64)(word32)n; + rax = (word64)(word32)qmask; + + x8 = _mm_cvtsi32_si128((int)(word32)rax); + x8 = _mm_broadcastw_epi16(x8); + r10 = (word64)((word64)(sword64)(sword32)(word32)r9); + r10 = (word64)(r10 >> 3); + r11 = (word64)((word64)(sword64)(sword32)(word32)r8); + rcx = (word64)((word64)(sword64)(sword32)(word32)rcx); + r12 = (word64)(0); +L_frodokem_gen_a_rows_aes_aesni_row: + r14 = (word64)(rcx); + r14 = (word64)(r14 + r12); + r13 = (word64)(0); +L_frodokem_gen_a_rows_aes_aesni_blk: + r15 = (word64)(r13 + 8); + if ((sword64)(r15) > (sword64)(r10)) { + goto L_frodokem_gen_a_rows_aes_aesni_tail; + } + x0 = _mm_cvtsi32_si128((int)(word32)r14); + r14 = (word64)(r14 + 0x80000); + x1 = _mm_cvtsi32_si128((int)(word32)r14); + r14 = (word64)(r14 + 0x80000); + x2 = _mm_cvtsi32_si128((int)(word32)r14); + r14 = (word64)(r14 + 0x80000); + x3 = _mm_cvtsi32_si128((int)(word32)r14); + r14 = (word64)(r14 + 0x80000); + x4 = _mm_cvtsi32_si128((int)(word32)r14); + r14 = (word64)(r14 + 0x80000); + x5 = _mm_cvtsi32_si128((int)(word32)r14); + r14 = (word64)(r14 + 0x80000); + x6 = _mm_cvtsi32_si128((int)(word32)r14); + r14 = (word64)(r14 + 0x80000); + x7 = _mm_cvtsi32_si128((int)(word32)r14); + r14 = (word64)(r14 + 0x80000); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x1 = _mm_xor_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x2 = _mm_xor_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x3 = _mm_xor_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x4 = _mm_xor_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x5 = _mm_xor_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x6 = _mm_xor_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x7 = _mm_xor_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x1 = _mm_aesenc_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x2 = _mm_aesenc_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x3 = _mm_aesenc_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x6 = _mm_aesenc_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x1 = _mm_aesenc_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x2 = _mm_aesenc_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x3 = _mm_aesenc_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x6 = _mm_aesenc_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x1 = _mm_aesenc_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x2 = _mm_aesenc_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x3 = _mm_aesenc_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x6 = _mm_aesenc_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x1 = _mm_aesenc_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x2 = _mm_aesenc_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x3 = _mm_aesenc_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x6 = _mm_aesenc_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x1 = _mm_aesenc_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x2 = _mm_aesenc_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x3 = _mm_aesenc_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x6 = _mm_aesenc_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x1 = _mm_aesenc_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x2 = _mm_aesenc_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x3 = _mm_aesenc_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x6 = _mm_aesenc_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x1 = _mm_aesenc_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x2 = _mm_aesenc_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x3 = _mm_aesenc_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x6 = _mm_aesenc_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x1 = _mm_aesenc_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x2 = _mm_aesenc_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x3 = _mm_aesenc_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x6 = _mm_aesenc_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x1 = _mm_aesenc_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x2 = _mm_aesenc_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x3 = _mm_aesenc_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x4 = _mm_aesenc_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x5 = _mm_aesenc_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x6 = _mm_aesenc_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x7 = _mm_aesenc_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x0 = _mm_aesenclast_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + x1 = _mm_aesenclast_si128(x1, _mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + x2 = _mm_aesenclast_si128(x2, _mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + x3 = _mm_aesenclast_si128(x3, _mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + x4 = _mm_aesenclast_si128(x4, _mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + x5 = _mm_aesenclast_si128(x5, _mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + x6 = _mm_aesenclast_si128(x6, _mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + x7 = _mm_aesenclast_si128(x7, _mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + x0 = _mm_and_si128(x0, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); + x1 = _mm_and_si128(x1, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 16), x1); + x2 = _mm_and_si128(x2, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 32), x2); + x3 = _mm_and_si128(x3, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 48), x3); + x4 = _mm_and_si128(x4, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 64), x4); + x5 = _mm_and_si128(x5, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 80), x5); + x6 = _mm_and_si128(x6, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 96), x6); + x7 = _mm_and_si128(x7, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 112), x7); + rsi = (word64)(rsi + 0x80); + r13 = (word64)(r13 + 8); + goto L_frodokem_gen_a_rows_aes_aesni_blk; +L_frodokem_gen_a_rows_aes_aesni_tail: + if ((sword64)(r13) >= (sword64)(r10)) { + goto L_frodokem_gen_a_rows_aes_aesni_next; + } + x0 = _mm_cvtsi32_si128((int)(word32)r14); + r14 = (word64)(r14 + 0x80000); + x0 = _mm_xor_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 16))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 32))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 48))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 64))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 80))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 96))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 112))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 128))); + x0 = _mm_aesenc_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, 144))); + x0 = _mm_aesenclast_si128(x0, _mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + x0 = _mm_and_si128(x0, x8); + _mm_storeu_si128((__m128i*)WC_PW(rsi, 0), x0); + rsi = (word64)(rsi + 0x10); + r13 = (word64)(r13 + 1); + goto L_frodokem_gen_a_rows_aes_aesni_tail; +L_frodokem_gen_a_rows_aes_aesni_next: + r12 = (word64)(r12 + 1); + if ((sword64)(r12) < (sword64)(r11)) { + goto L_frodokem_gen_a_rows_aes_aesni_row; + } + (void)rdi; +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* WOLFSSL_HAVE_FRODOKEM */ +#ifdef WOLFSSL_HAVE_FRODOKEM +#ifdef HAVE_INTEL_AVX512 +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void frodokem_sa_accum_avx512(word16* out, const word16* s, + const word16* row, int j, int n) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11, r12, r13, r14, r15, rbp, + rbx = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), + z8 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)s; + rdx = (word64)(size_t)row; + rcx = (word64)(word32)j; + r8 = (word64)(word32)n; + + rcx = (word64)((word64)(sword64)(sword32)(word32)rcx); + rcx = (word64)(rcx << 1); + rsi = (word64)(rsi + rcx); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + rax = (word64)(rax << 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & -64); + r9 = (word64)(rdx); + r9 = (word64)(r9 + rax); + r10 = (word64)(r9); + r10 = (word64)(r10 + rax); + r11 = (word64)(r10); + r11 = (word64)(r11 + rax); + r12 = (word64)(r11); + r12 = (word64)(r12 + rax); + r13 = (word64)(r12); + r13 = (word64)(r13 + rax); + r14 = (word64)(r13); + r14 = (word64)(r14 + rax); + r15 = (word64)(r14); + r15 = (word64)(r15 + rax); + rbp = (word64)(8); +L_frodokem_sa_accum_avx512_i: + z0 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 0))); + z1 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 2))); + z2 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 4))); + z3 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 6))); + z4 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 8))); + z5 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 10))); + z6 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 12))); + z7 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 14))); + rbx = (word64)(0); +L_frodokem_sa_accum_avx512_k: + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, rbx)); + z9 = _mm512_mullo_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + rbx))); + z8 = _mm512_add_epi16(z8, z9); + z9 = _mm512_mullo_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(r9, + rbx))); + z8 = _mm512_add_epi16(z8, z9); + z9 = _mm512_mullo_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(r10, + rbx))); + z8 = _mm512_add_epi16(z8, z9); + z9 = _mm512_mullo_epi16(z3, _mm512_loadu_si512((const void*)WC_PR(r11, + rbx))); + z8 = _mm512_add_epi16(z8, z9); + z9 = _mm512_mullo_epi16(z4, _mm512_loadu_si512((const void*)WC_PR(r12, + rbx))); + z8 = _mm512_add_epi16(z8, z9); + z9 = _mm512_mullo_epi16(z5, _mm512_loadu_si512((const void*)WC_PR(r13, + rbx))); + z8 = _mm512_add_epi16(z8, z9); + z9 = _mm512_mullo_epi16(z6, _mm512_loadu_si512((const void*)WC_PR(r14, + rbx))); + z8 = _mm512_add_epi16(z8, z9); + z9 = _mm512_mullo_epi16(z7, _mm512_loadu_si512((const void*)WC_PR(r15, + rbx))); + z8 = _mm512_add_epi16(z8, z9); + _mm512_storeu_si512((void*)WC_PW(rdi, rbx), z8); + rbx = (word64)(rbx + 0x40); + if ((sword64)(rbx) < (sword64)(rcx)) { + goto L_frodokem_sa_accum_avx512_k; + } + if ((sword64)(rbx) >= (sword64)(rax)) { + goto L_frodokem_sa_accum_avx512_tail; + } + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + rbx))); + z9 = _mm512_zextsi256_si512(_mm256_mullo_epi16(_mm512_castsi512_si256(z0), + _mm256_loadu_si256((const __m256i*)WC_PR(rdx, rbx)))); + z8 = _mm512_zextsi256_si512(_mm256_add_epi16(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9))); + z9 = _mm512_zextsi256_si512(_mm256_mullo_epi16(_mm512_castsi512_si256(z1), + _mm256_loadu_si256((const __m256i*)WC_PR(r9, rbx)))); + z8 = _mm512_zextsi256_si512(_mm256_add_epi16(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9))); + z9 = _mm512_zextsi256_si512(_mm256_mullo_epi16(_mm512_castsi512_si256(z2), + _mm256_loadu_si256((const __m256i*)WC_PR(r10, rbx)))); + z8 = _mm512_zextsi256_si512(_mm256_add_epi16(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9))); + z9 = _mm512_zextsi256_si512(_mm256_mullo_epi16(_mm512_castsi512_si256(z3), + _mm256_loadu_si256((const __m256i*)WC_PR(r11, rbx)))); + z8 = _mm512_zextsi256_si512(_mm256_add_epi16(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9))); + z9 = _mm512_zextsi256_si512(_mm256_mullo_epi16(_mm512_castsi512_si256(z4), + _mm256_loadu_si256((const __m256i*)WC_PR(r12, rbx)))); + z8 = _mm512_zextsi256_si512(_mm256_add_epi16(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9))); + z9 = _mm512_zextsi256_si512(_mm256_mullo_epi16(_mm512_castsi512_si256(z5), + _mm256_loadu_si256((const __m256i*)WC_PR(r13, rbx)))); + z8 = _mm512_zextsi256_si512(_mm256_add_epi16(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9))); + z9 = _mm512_zextsi256_si512(_mm256_mullo_epi16(_mm512_castsi512_si256(z6), + _mm256_loadu_si256((const __m256i*)WC_PR(r14, rbx)))); + z8 = _mm512_zextsi256_si512(_mm256_add_epi16(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9))); + z9 = _mm512_zextsi256_si512(_mm256_mullo_epi16(_mm512_castsi512_si256(z7), + _mm256_loadu_si256((const __m256i*)WC_PR(r15, rbx)))); + z8 = _mm512_zextsi256_si512(_mm256_add_epi16(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9))); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, rbx), _mm512_castsi512_si256(z8)); +L_frodokem_sa_accum_avx512_tail: + rdi = (word64)(rdi + rax); + rsi = (word64)(rsi + rax); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_frodokem_sa_accum_avx512_i; + } +} + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void frodokem_as_accum_avx512(word16* out, const word16* s, + const word16* row, int i, int n) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11, r12, r13, r14, r15, rbp, + rbx = 0; + __m128i x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13 = _mm_setzero_si128(); + __m256i y22 = _mm256_setzero_si256(), y23 = _mm256_setzero_si256(), + y24 = _mm256_setzero_si256(), y25 = _mm256_setzero_si256(), + y26 = _mm256_setzero_si256(), y27 = _mm256_setzero_si256(), + y28 = _mm256_setzero_si256(), y29 = _mm256_setzero_si256(); + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z14 = _mm512_setzero_si512(), z15 = _mm512_setzero_si512(), + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)s; + rdx = (word64)(size_t)row; + rcx = (word64)(word32)i; + r8 = (word64)(word32)n; + + rcx = (word64)((word64)(sword64)(sword32)(word32)rcx); + rcx = (word64)(rcx << 4); + rdi = (word64)(rdi + rcx); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + rax = (word64)(rax << 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & -64); + r9 = (word64)(rsi); + r9 = (word64)(r9 + rax); + r10 = (word64)(r9); + r10 = (word64)(r10 + rax); + r11 = (word64)(r10); + r11 = (word64)(r11 + rax); + r12 = (word64)(r11); + r12 = (word64)(r12 + rax); + r13 = (word64)(r12); + r13 = (word64)(r13 + rax); + r14 = (word64)(r13); + r14 = (word64)(r14 + rax); + r15 = (word64)(r14); + r15 = (word64)(r15 + rax); + rbp = (word64)(8); + x13 = _mm_cmpeq_epi32(x13, x13); + x13 = _mm_srli_epi32(x13, 16); +L_frodokem_as_accum_avx512_r: + z14 = _mm512_setzero_si512(); + y22 = _mm256_setzero_si256(); + z15 = _mm512_setzero_si512(); + y23 = _mm256_setzero_si256(); + z16 = _mm512_setzero_si512(); + y24 = _mm256_setzero_si256(); + z17 = _mm512_setzero_si512(); + y25 = _mm256_setzero_si256(); + z18 = _mm512_setzero_si512(); + y26 = _mm256_setzero_si256(); + z19 = _mm512_setzero_si512(); + y27 = _mm256_setzero_si256(); + z20 = _mm512_setzero_si512(); + y28 = _mm256_setzero_si256(); + z21 = _mm512_setzero_si512(); + y29 = _mm256_setzero_si256(); + rbx = (word64)(0); +L_frodokem_as_accum_avx512_j: + z0 = _mm512_loadu_si512((const void*)WC_PR(rdx, rbx)); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + rbx))); + z14 = _mm512_add_epi32(z14, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r9, rbx))); + z15 = _mm512_add_epi32(z15, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r10, + rbx))); + z16 = _mm512_add_epi32(z16, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r11, + rbx))); + z17 = _mm512_add_epi32(z17, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r12, + rbx))); + z18 = _mm512_add_epi32(z18, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r13, + rbx))); + z19 = _mm512_add_epi32(z19, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r14, + rbx))); + z20 = _mm512_add_epi32(z20, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r15, + rbx))); + z21 = _mm512_add_epi32(z21, z1); + rbx = (word64)(rbx + 0x40); + if ((sword64)(rbx) < (sword64)(rcx)) { + goto L_frodokem_as_accum_avx512_j; + } + if ((sword64)(rbx) >= (sword64)(rax)) { + goto L_frodokem_as_accum_avx512_tail; + } + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + rbx))); + y22 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, rbx))); + y23 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r9, rbx))); + y24 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r10, rbx))); + y25 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r11, rbx))); + y26 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r12, rbx))); + y27 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r13, rbx))); + y28 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r14, rbx))); + y29 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r15, rbx))); +L_frodokem_as_accum_avx512_tail: + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z14, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z14), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y22)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x2 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z15, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z15), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y23)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x3 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z16, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z16), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y24)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x4 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z17, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z17), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y25)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x5 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z18, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z18), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y26)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x6 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z19, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z19), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y27)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x7 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z20, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z20), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y28)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x8 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z21, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z21), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y29)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x9 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + x11 = _mm_hadd_epi32(x2, x3); + x12 = _mm_hadd_epi32(x4, x5); + x10 = _mm_hadd_epi32(x11, x12); + x11 = _mm_hadd_epi32(x6, x7); + x12 = _mm_hadd_epi32(x8, x9); + x11 = _mm_hadd_epi32(x11, x12); + x10 = _mm_and_si128(x10, x13); + x11 = _mm_and_si128(x11, x13); + x10 = _mm_packus_epi32(x10, x11); + x10 = _mm_add_epi16(x10, _mm_loadu_si128((const __m128i*)WC_PR(rdi, 0))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x10); + rdx = (word64)(rdx + rax); + rdi = (word64)(rdi + 0x10); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_frodokem_as_accum_avx512_r; + } +} + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void frodokem_mul_bs_avx512(word16* out, const word16* b, + const word16* s, int n, int qmask) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11, r12, r13, r14, r15, rbp, + rbx = 0; + __m128i x2 = _mm_setzero_si128(), x3 = _mm_setzero_si128(), + x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(), + x6 = _mm_setzero_si128(), x7 = _mm_setzero_si128(), + x8 = _mm_setzero_si128(), x9 = _mm_setzero_si128(), + x10 = _mm_setzero_si128(), x11 = _mm_setzero_si128(), + x12 = _mm_setzero_si128(), x13; + __m256i y22 = _mm256_setzero_si256(), y23 = _mm256_setzero_si256(), + y24 = _mm256_setzero_si256(), y25 = _mm256_setzero_si256(), + y26 = _mm256_setzero_si256(), y27 = _mm256_setzero_si256(), + y28 = _mm256_setzero_si256(), y29 = _mm256_setzero_si256(); + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z14 = _mm512_setzero_si512(), z15 = _mm512_setzero_si512(), + z16 = _mm512_setzero_si512(), z17 = _mm512_setzero_si512(), + z18 = _mm512_setzero_si512(), z19 = _mm512_setzero_si512(), + z20 = _mm512_setzero_si512(), z21 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)b; + rdx = (word64)(size_t)s; + rcx = (word64)(word32)n; + r8 = (word64)(word32)qmask; + + rax = (word64)((word64)(sword64)(sword32)(word32)rcx); + rax = (word64)(rax << 1); + rcx = (word64)(rax); + rcx = (word64)(rcx & -64); + r9 = (word64)(rdx); + r9 = (word64)(r9 + rax); + r10 = (word64)(r9); + r10 = (word64)(r10 + rax); + r11 = (word64)(r10); + r11 = (word64)(r11 + rax); + r12 = (word64)(r11); + r12 = (word64)(r12 + rax); + r13 = (word64)(r12); + r13 = (word64)(r13 + rax); + r14 = (word64)(r13); + r14 = (word64)(r14 + rax); + r15 = (word64)(r14); + r15 = (word64)(r15 + rax); + rbp = (word64)(8); + x13 = _mm_cvtsi32_si128((int)(word32)r8); + x13 = _mm_broadcastd_epi32(x13); +L_frodokem_mul_bs_avx512_r: + z14 = _mm512_setzero_si512(); + y22 = _mm256_setzero_si256(); + z15 = _mm512_setzero_si512(); + y23 = _mm256_setzero_si256(); + z16 = _mm512_setzero_si512(); + y24 = _mm256_setzero_si256(); + z17 = _mm512_setzero_si512(); + y25 = _mm256_setzero_si256(); + z18 = _mm512_setzero_si512(); + y26 = _mm256_setzero_si256(); + z19 = _mm512_setzero_si512(); + y27 = _mm256_setzero_si256(); + z20 = _mm512_setzero_si512(); + y28 = _mm256_setzero_si256(); + z21 = _mm512_setzero_si512(); + y29 = _mm256_setzero_si256(); + rbx = (word64)(0); +L_frodokem_mul_bs_avx512_j: + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, rbx)); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rdx, + rbx))); + z14 = _mm512_add_epi32(z14, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r9, rbx))); + z15 = _mm512_add_epi32(z15, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r10, + rbx))); + z16 = _mm512_add_epi32(z16, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r11, + rbx))); + z17 = _mm512_add_epi32(z17, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r12, + rbx))); + z18 = _mm512_add_epi32(z18, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r13, + rbx))); + z19 = _mm512_add_epi32(z19, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r14, + rbx))); + z20 = _mm512_add_epi32(z20, z1); + z1 = _mm512_madd_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(r15, + rbx))); + z21 = _mm512_add_epi32(z21, z1); + rbx = (word64)(rbx + 0x40); + if ((sword64)(rbx) < (sword64)(rcx)) { + goto L_frodokem_mul_bs_avx512_j; + } + if ((sword64)(rbx) >= (sword64)(rax)) { + goto L_frodokem_mul_bs_avx512_tail; + } + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + rbx))); + y22 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(rdx, rbx))); + y23 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r9, rbx))); + y24 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r10, rbx))); + y25 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r11, rbx))); + y26 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r12, rbx))); + y27 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r13, rbx))); + y28 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r14, rbx))); + y29 = _mm256_madd_epi16(_mm512_castsi512_si256(z0), _mm256_loadu_si256(( + const __m256i*)WC_PR(r15, rbx))); +L_frodokem_mul_bs_avx512_tail: + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z14, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z14), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y22)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x2 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z15, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z15), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y23)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x3 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z16, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z16), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y24)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x4 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z17, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z17), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y25)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x5 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z18, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z18), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y26)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x6 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z19, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z19), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y27)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x7 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z20, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z20), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y28)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x8 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + z1 = _mm512_zextsi256_si512(_mm512_extracti64x4_epi64(z21, 1)); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z21), + _mm512_castsi512_si256(z1))); + z1 = _mm512_zextsi256_si512(_mm256_add_epi32(_mm512_castsi512_si256(z1), + y29)); + z0 = _mm512_zextsi128_si512(_mm256_extracti128_si256(_mm512_castsi512_si256( + z1), 1)); + x9 = _mm_add_epi32(_mm512_castsi512_si128(z1), _mm512_castsi512_si128(z0)); + x11 = _mm_hadd_epi32(x2, x3); + x12 = _mm_hadd_epi32(x4, x5); + x10 = _mm_hadd_epi32(x11, x12); + x11 = _mm_hadd_epi32(x6, x7); + x12 = _mm_hadd_epi32(x8, x9); + x11 = _mm_hadd_epi32(x11, x12); + x10 = _mm_and_si128(x10, x13); + x11 = _mm_and_si128(x11, x13); + x10 = _mm_packus_epi32(x10, x11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), x10); + rsi = (word64)(rsi + rax); + rdi = (word64)(rdi + 0x10); + rbp = (word64)(rbp - 1); + if ((rbp) != (0)) { + goto L_frodokem_mul_bs_avx512_r; + } +} + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e_avx512(word16* out, + const word16* b, const word16* s, int n, int qmask) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11, r12, r13, r14, r15, rbx; + __m512i z0, z1, z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5; + + rdi = (word64)(size_t)out; + rsi = (word64)(size_t)b; + rdx = (word64)(size_t)s; + rcx = (word64)(word32)n; + r8 = (word64)(word32)qmask; + + rax = (word64)((word64)(sword64)(sword32)(word32)rcx); + rax = (word64)(rax << 1); + r9 = (word64)(rdx); + r9 = (word64)(r9 + rax); + r10 = (word64)(r9); + r10 = (word64)(r10 + rax); + r11 = (word64)(r10); + r11 = (word64)(r11 + rax); + r12 = (word64)(r11); + r12 = (word64)(r12 + rax); + r13 = (word64)(r12); + r13 = (word64)(r13 + rax); + r14 = (word64)(r13); + r14 = (word64)(r14 + rax); + r15 = (word64)(r14); + r15 = (word64)(r15 + rax); + z5 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r8)); + z5 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z5)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + rbx = (word64)(0); +L_frodokem_mul_add_sb_plus_e_avx512_j: + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + z3 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + rbx))); + z4 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r9, + rbx))); + z3 = _mm512_inserti32x4(z3, _mm512_castsi512_si128(z4), 1); + z4 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r10, + rbx))); + z3 = _mm512_inserti32x4(z3, _mm512_castsi512_si128(z4), 2); + z4 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r11, + rbx))); + z3 = _mm512_inserti32x4(z3, _mm512_castsi512_si128(z4), 3); + z4 = _mm512_mullo_epi16(z3, z2); + z0 = _mm512_add_epi16(z0, z4); + z3 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r12, + rbx))); + z4 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + z3 = _mm512_inserti32x4(z3, _mm512_castsi512_si128(z4), 1); + z4 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r14, + rbx))); + z3 = _mm512_inserti32x4(z3, _mm512_castsi512_si128(z4), 2); + z4 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r15, + rbx))); + z3 = _mm512_inserti32x4(z3, _mm512_castsi512_si128(z4), 3); + z4 = _mm512_mullo_epi16(z3, z2); + z1 = _mm512_add_epi16(z1, z4); + rsi = (word64)(rsi + 0x10); + rbx = (word64)(rbx + 2); + if ((sword64)(rbx) < (sword64)(rax)) { + goto L_frodokem_mul_add_sb_plus_e_avx512_j; + } + z0 = _mm512_and_si512(z0, z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + z1 = _mm512_and_si512(z1, z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); +} + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void frodokem_add_avx512(word16* a, const word16* b, int qmask) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + rdx = (word64)(word32)qmask; + + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_add_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, 0))); + z1 = _mm512_and_si512(z1, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z1); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_add_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(rsi, 64))); + z2 = _mm512_and_si512(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z2); +} + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void frodokem_sample_avx512(word16* mat, int cnt, + const word16* cdf, int cdflen) +{ + word64 rdi, rsi, rdx, rcx, rax, r8, r9 = 0, r10 = 0; + __m512i z0 = _mm512_setzero_si512(), z1, z2 = _mm512_setzero_si512(), + z3 = _mm512_setzero_si512(), z4 = _mm512_setzero_si512(), + z5 = _mm512_setzero_si512(), z6 = _mm512_setzero_si512(), + z7 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)mat; + rsi = (word64)(word32)cnt; + rdx = (word64)(size_t)cdf; + rcx = (word64)(word32)cdflen; + + rax = (word64)((word64)(sword64)(sword32)(word32)rsi); + rax = (word64)(rax << 1); + z0 = _mm512_ternarylogic_epi32(z0, z0, z0, 0xff); + z0 = _mm512_srli_epi16(z0, 15); + z1 = _mm512_setzero_si512(); + r8 = (word64)(0); +L_frodokem_sample_avx512_blk: + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, r8)); + z3 = _mm512_srli_epi16(z2, 1); + z4 = _mm512_and_si512(z2, z0); + z5 = _mm512_setzero_si512(); + r9 = (word64)(rdx); + r10 = (word32)((word32)rcx); +L_frodokem_sample_avx512_cdf: + z6 = _mm512_broadcastw_epi16(_mm_loadu_si128((const __m128i*)WC_PR(r9, 0))); + z6 = _mm512_sub_epi16(z6, z3); + z6 = _mm512_srli_epi16(z6, 15); + z5 = _mm512_add_epi16(z5, z6); + r9 = (word64)(r9 + 2); + r10 = (word64)(r10 - 1); + if ((r10) != (0)) { + goto L_frodokem_sample_avx512_cdf; + } + z7 = _mm512_sub_epi16(z1, z4); + z5 = _mm512_xor_si512(z5, z7); + z5 = _mm512_add_epi16(z5, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, r8), z5); + r8 = (word64)(r8 + 0x40); + if ((sword64)(r8) < (sword64)(rax)) { + goto L_frodokem_sample_avx512_blk; + } +} + +WC_X64I_TARGET("vaes,avx512f,avx512bw") +WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_avx512(byte* in, word16* out, + const byte* ks, int i, int cnt, int n, int qmask) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11, r15, r12, r14 = 0, + r13 = 0; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, + z11 = _mm512_setzero_si512(), z12 = _mm512_setzero_si512(), + z13 = _mm512_setzero_si512(), z14 = _mm512_setzero_si512(), z15; + + rdi = (word64)(size_t)in; + rsi = (word64)(size_t)out; + rdx = (word64)(size_t)ks; + rcx = (word64)(word32)i; + r8 = (word64)(word32)cnt; + r9 = (word64)(word32)n; + rax = (word64)(word32)qmask; + + z0 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + z1 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 16))); + z2 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 32))); + z3 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 48))); + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 64))); + z5 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 80))); + z6 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 96))); + z7 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 112))); + z8 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 128))); + z9 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 144))); + z10 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rdx, + 160))); + z15 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z15 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z15)); + r10 = (word64)((word64)(sword64)(sword32)(word32)r9); + r10 = (word64)(r10 << 1); + r11 = (word64)((word64)(sword64)(sword32)(word32)r8); + r11 = (word64)(r11 * r10); + rcx = (word64)((word64)(sword64)(sword32)(word32)rcx); + r15 = (word64)(0); + r12 = (word64)(rdi); +L_frodokem_gen_a_rows_aes_avx512_row: + r14 = (word32)((word32)rcx); + r13 = (word64)(r12); + r13 = (word64)(r13 + r10); +L_frodokem_gen_a_rows_aes_avx512_blk: + WC_S64(r12, 0) = (word64)(r14); + WC_S64(r12, 8) = (word64)(r15); + r14 = (word64)(r14 + 0x80000); + r12 = (word64)(r12 + 0x10); + if ((sword64)(r12) < (sword64)(r13)) { + goto L_frodokem_gen_a_rows_aes_avx512_blk; + } + rcx = (word64)(rcx + 1); + r13 = (word64)(rdi); + r13 = (word64)(r13 + r11); + if ((sword64)(r12) < (sword64)(r13)) { + goto L_frodokem_gen_a_rows_aes_avx512_row; + } + r14 = (word64)(0); +L_frodokem_gen_a_rows_aes_avx512_aes: + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z11 = _mm512_xor_si512(z11, z0); + z12 = _mm512_xor_si512(z12, z0); + z13 = _mm512_xor_si512(z13, z0); + z14 = _mm512_xor_si512(z14, z0); + z11 = _mm512_aesenc_epi128(z11, z1); + z12 = _mm512_aesenc_epi128(z12, z1); + z13 = _mm512_aesenc_epi128(z13, z1); + z14 = _mm512_aesenc_epi128(z14, z1); + z11 = _mm512_aesenc_epi128(z11, z2); + z12 = _mm512_aesenc_epi128(z12, z2); + z13 = _mm512_aesenc_epi128(z13, z2); + z14 = _mm512_aesenc_epi128(z14, z2); + z11 = _mm512_aesenc_epi128(z11, z3); + z12 = _mm512_aesenc_epi128(z12, z3); + z13 = _mm512_aesenc_epi128(z13, z3); + z14 = _mm512_aesenc_epi128(z14, z3); + z11 = _mm512_aesenc_epi128(z11, z4); + z12 = _mm512_aesenc_epi128(z12, z4); + z13 = _mm512_aesenc_epi128(z13, z4); + z14 = _mm512_aesenc_epi128(z14, z4); + z11 = _mm512_aesenc_epi128(z11, z5); + z12 = _mm512_aesenc_epi128(z12, z5); + z13 = _mm512_aesenc_epi128(z13, z5); + z14 = _mm512_aesenc_epi128(z14, z5); + z11 = _mm512_aesenc_epi128(z11, z6); + z12 = _mm512_aesenc_epi128(z12, z6); + z13 = _mm512_aesenc_epi128(z13, z6); + z14 = _mm512_aesenc_epi128(z14, z6); + z11 = _mm512_aesenc_epi128(z11, z7); + z12 = _mm512_aesenc_epi128(z12, z7); + z13 = _mm512_aesenc_epi128(z13, z7); + z14 = _mm512_aesenc_epi128(z14, z7); + z11 = _mm512_aesenc_epi128(z11, z8); + z12 = _mm512_aesenc_epi128(z12, z8); + z13 = _mm512_aesenc_epi128(z13, z8); + z14 = _mm512_aesenc_epi128(z14, z8); + z11 = _mm512_aesenc_epi128(z11, z9); + z12 = _mm512_aesenc_epi128(z12, z9); + z13 = _mm512_aesenc_epi128(z13, z9); + z14 = _mm512_aesenc_epi128(z14, z9); + z11 = _mm512_aesenclast_epi128(z11, z10); + z12 = _mm512_aesenclast_epi128(z12, z10); + z13 = _mm512_aesenclast_epi128(z13, z10); + z14 = _mm512_aesenclast_epi128(z14, z10); + z11 = _mm512_and_si512(z11, z15); + z12 = _mm512_and_si512(z12, z15); + z13 = _mm512_and_si512(z13, z15); + z14 = _mm512_and_si512(z14, z15); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z12); + _mm512_storeu_si512((void*)WC_PW(rsi, 128), z13); + _mm512_storeu_si512((void*)WC_PW(rsi, 192), z14); + rdi = (word64)(rdi + 0x100); + rsi = (word64)(rsi + 0x100); + r14 = (word64)(r14 + 0x100); + if ((sword64)(r14) < (sword64)(r11)) { + goto L_frodokem_gen_a_rows_aes_avx512_aes; + } +} + +#endif /* HAVE_INTEL_AVX512 */ +#endif /* WOLFSSL_HAVE_FRODOKEM */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/wc_mldsa_intrin.c b/wolfcrypt/src/wc_mldsa_intrin.c new file mode 100644 index 00000000000..0f9661b7737 --- /dev/null +++ b/wolfcrypt/src/wc_mldsa_intrin.c @@ -0,0 +1,64033 @@ +/* wc_mldsa_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_WC_MLDSA_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_HAVE_MLDSA +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void wc_mldsa_poly_red_avx2(sword32* a); +extern WOLFSSL_LOCAL void wc_mldsa_ntt_avx2(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_ntt_small_avx2(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_ntt_full_avx2(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_ntt_small_full_avx2(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_invntt_avx2(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_invntt_full_avx2(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_mul_avx2(sword32* r, const sword32* a, + const sword32* b); +extern WOLFSSL_LOCAL void wc_mldsa_mul_vec_4_avx2(sword32* r, const sword32* a, + const sword32* b); +extern WOLFSSL_LOCAL void wc_mldsa_mul_vec_5_avx2(sword32* r, const sword32* a, + const sword32* b); +extern WOLFSSL_LOCAL void wc_mldsa_mul_vec_7_avx2(sword32* r, const sword32* a, + const sword32* b); +extern WOLFSSL_LOCAL int wc_mldsa_rej_uniform_n_avx2(sword32* p, word32 len, + const byte* r, word32 rLen); +extern WOLFSSL_LOCAL int wc_mldsa_rej_uniform_avx2(sword32* p, word32 len, + const byte* r, word32 rLen); +extern WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta2_avx2(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt); +extern WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta4_avx2(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt); +extern WOLFSSL_LOCAL void wc_mldsa_redistribute_21_rand_avx2(word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +extern WOLFSSL_LOCAL void wc_mldsa_redistribute_17_rand_avx2(word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +extern WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_2_avx2(const sword32* s, + byte k, byte* p); +extern WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_4_avx2(const sword32* s, + byte* p); +extern WOLFSSL_LOCAL void wc_mldsa_decode_eta_2_avx2(const byte* p, sword32* s); +extern WOLFSSL_LOCAL void wc_mldsa_decode_eta_4_avx2(const byte* p, sword32* s); +extern WOLFSSL_LOCAL void wc_mldsa_encode_w1_88_avx2(const sword32* w1, + byte* w1e); +extern WOLFSSL_LOCAL void wc_mldsa_encode_w1_32_avx2(const sword32* w1, + byte* w1e); +extern WOLFSSL_LOCAL void wc_mldsa_vec_encode_t0_t1_avx2(const sword32* t, + byte k, byte* t0, byte* t1); +extern WOLFSSL_LOCAL void wc_mldsa_decode_t0_avx2(const byte* t0, sword32* t); +extern WOLFSSL_LOCAL void wc_mldsa_decode_t1_avx2(const byte* t1, sword32* t); +extern WOLFSSL_LOCAL void wc_mldsa_decode_gamma1_17_avx2(const byte* s, + sword32* z); +extern WOLFSSL_LOCAL void wc_mldsa_decode_gamma1_19_avx2(const byte* s, + sword32* z); +extern WOLFSSL_LOCAL void wc_mldsa_encode_gamma1_17_avx2(const sword32* z, + byte* s); +extern WOLFSSL_LOCAL void wc_mldsa_encode_gamma1_19_avx2(const sword32* z, + byte* s); +extern WOLFSSL_LOCAL void wc_mldsa_decompose_q88_avx2(const sword32* r, + sword32* r0, sword32* r1); +extern WOLFSSL_LOCAL void wc_mldsa_decompose_q32_avx2(const sword32* r, byte k, + sword32* r0, sword32* r1); +extern WOLFSSL_LOCAL void wc_mldsa_use_hint_88_avx2(sword32* w1, const byte* h); +extern WOLFSSL_LOCAL void wc_mldsa_use_hint_32_avx2(sword32* w1, byte k, + const byte* h); +extern WOLFSSL_LOCAL int wc_mldsa_vec_check_low_avx2(const sword32* a, byte l, + sword32 hi); +extern WOLFSSL_LOCAL void wc_mldsa_poly_add_avx2(sword32* r, const sword32* a); +extern WOLFSSL_LOCAL void wc_mldsa_poly_sub_avx2(sword32* r, const sword32* a); +extern WOLFSSL_LOCAL void wc_mldsa_poly_make_pos_avx2(sword32* a); +#endif +#ifdef HAVE_INTEL_AVX512 +extern WOLFSSL_LOCAL int wc_mldsa_rej_uniform_n_avx512(sword32* p, word32 len, + const byte* r, word32 rLen); +extern WOLFSSL_LOCAL int wc_mldsa_rej_uniform_avx512(sword32* p, word32 len, + const byte* r, word32 rLen); +extern WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta2_avx512(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt); +extern WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta4_avx512(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt); +extern WOLFSSL_LOCAL int wc_mldsa_make_hint_88_avx512(const sword32* s, + const sword32* w1, unsigned int omega, byte* h, byte* idx); +extern WOLFSSL_LOCAL int wc_mldsa_make_hint_32_avx512(const sword32* s, + const sword32* w1, unsigned int omega, byte* h, byte* idx); +extern WOLFSSL_LOCAL void wc_mldsa_use_hint_88_avx512(sword32* w1, + const byte* h); +extern WOLFSSL_LOCAL void wc_mldsa_use_hint_32_avx512(sword32* w1, byte k, + const byte* h); +extern WOLFSSL_LOCAL void wc_mldsa_redistribute_21_rand_x8_avx512( + const word64* s, byte* out, word32 stride); +extern WOLFSSL_LOCAL void wc_mldsa_redistribute_17_rand_x8_avx512( + const word64* s, byte* out, word32 stride); +extern WOLFSSL_LOCAL void wc_mldsa_decode_gamma1_17_x2_avx512(const byte* sA, + const byte* sB, sword32* zA, sword32* zB); +extern WOLFSSL_LOCAL void wc_mldsa_decode_gamma1_19_x2_avx512(const byte* sA, + const byte* sB, sword32* zA, sword32* zB); +extern WOLFSSL_LOCAL void wc_mldsa_decode_t0_x2_avx512(const byte* t0A, + const byte* t0B, sword32* tA, sword32* tB); +extern WOLFSSL_LOCAL void wc_mldsa_decode_t1_x2_avx512(const byte* t1A, + const byte* t1B, sword32* tA, sword32* tB); +extern WOLFSSL_LOCAL void wc_mldsa_decode_eta_2_x2_avx512(const byte* pA, + const byte* pB, sword32* sA, sword32* sB); +extern WOLFSSL_LOCAL void wc_mldsa_decode_eta_4_x2_avx512(const byte* pA, + const byte* pB, sword32* sA, sword32* sB); +extern WOLFSSL_LOCAL void wc_mldsa_encode_gamma1_17_avx512(const sword32* z, + byte* s); +extern WOLFSSL_LOCAL void wc_mldsa_encode_gamma1_19_avx512(const sword32* z, + byte* s); +extern WOLFSSL_LOCAL void wc_mldsa_encode_w1_88_x2_avx512(const sword32* w1A, + const sword32* w1B, byte* w1eA, byte* w1eB); +extern WOLFSSL_LOCAL void wc_mldsa_encode_w1_32_avx512(const sword32* w1, + byte* w1e); +extern WOLFSSL_LOCAL void wc_mldsa_encode_eta_2_x2_avx512(const sword32* sA, + const sword32* sB, byte* pA, byte* pB); +extern WOLFSSL_LOCAL void wc_mldsa_encode_eta_4_avx512(const sword32* s, + byte* p); +extern WOLFSSL_LOCAL void wc_mldsa_encode_t0_t1_x2_avx512(const sword32* tA, + const sword32* tB, byte* t0A, byte* t0B, byte* t1A, byte* t1B); +extern WOLFSSL_LOCAL void wc_mldsa_ntt_1p_avx512(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_ntt_small_1p_avx512(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_ntt_full_1p_avx512(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_ntt_small_full_1p_avx512(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_invntt_1p_avx512(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_invntt_full_1p_avx512(sword32* r); +extern WOLFSSL_LOCAL void wc_mldsa_poly_red_avx512(sword32* a); +extern WOLFSSL_LOCAL void wc_mldsa_mul_avx512(sword32* r, const sword32* a, + const sword32* b); +extern WOLFSSL_LOCAL void wc_mldsa_mul_vec_4_avx512(sword32* r, + const sword32* a, const sword32* b); +extern WOLFSSL_LOCAL void wc_mldsa_mul_vec_5_avx512(sword32* r, + const sword32* a, const sword32* b); +extern WOLFSSL_LOCAL void wc_mldsa_mul_vec_7_avx512(sword32* r, + const sword32* a, const sword32* b); +extern WOLFSSL_LOCAL void wc_mldsa_decompose_q88_avx512(const sword32* r, + sword32* r0, sword32* r1); +extern WOLFSSL_LOCAL void wc_mldsa_decompose_q32_avx512(const sword32* r, + byte k, sword32* r0, sword32* r1); +extern WOLFSSL_LOCAL int wc_mldsa_vec_check_low_avx512(const sword32* a, byte l, + sword32 hi); +extern WOLFSSL_LOCAL void wc_mldsa_poly_add_avx512(sword32* r, + const sword32* a); +extern WOLFSSL_LOCAL void wc_mldsa_poly_sub_avx512(sword32* r, + const sword32* a); +extern WOLFSSL_LOCAL void wc_mldsa_poly_make_pos_avx512(sword32* a); +#endif +#ifdef HAVE_INTEL_AVX512_VBMI +extern WOLFSSL_LOCAL void wc_mldsa_encode_w1_88_avx512_vbmi(const sword32* w1, + byte* w1e); +extern WOLFSSL_LOCAL void wc_mldsa_encode_t0_t1_avx512_vbmi(const sword32* t, + byte* t0, byte* t1); + +#endif +#endif +#ifdef WOLFSSL_HAVE_MLDSA +#ifdef HAVE_INTEL_AVX2 +XALIGNED(16) static const word32 mldsa_q[] WC_X64I_UNUSED = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +XALIGNED(16) static const word32 mldsa_qinv[] WC_X64I_UNUSED = { + 0x03802001, 0x03802001, 0x03802001, 0x03802001, + 0x03802001, 0x03802001, 0x03802001, 0x03802001, +}; + +XALIGNED(16) static const word32 mldsa_v[] WC_X64I_UNUSED = { + 0x00400000, 0x00400000, 0x00400000, 0x00400000, + 0x00400000, 0x00400000, 0x00400000, 0x00400000, +}; + +XALIGNED(16) static const word32 L_mldsa_avx2_zetas[] WC_X64I_UNUSED = { + 0x000064f7, 0x000064f7, 0x000064f7, 0x000064f7, + 0x000064f7, 0x000064f7, 0x000064f7, 0x000064f7, + 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, + 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, + 0xffd83102, 0xffd83102, 0xffd83102, 0xffd83102, + 0xffd83102, 0xffd83102, 0xffd83102, 0xffd83102, + 0x8cf87102, 0x8cf87102, 0x8cf87102, 0x8cf87102, + 0x8cf87102, 0x8cf87102, 0x8cf87102, 0x8cf87102, + 0xfff81503, 0xfff81503, 0xfff81503, 0xfff81503, + 0xfff81503, 0xfff81503, 0xfff81503, 0xfff81503, + 0x8d187503, 0x8d187503, 0x8d187503, 0x8d187503, + 0x8d187503, 0x8d187503, 0x8d187503, 0x8d187503, + 0x00039e44, 0x00039e44, 0x00039e44, 0x00039e44, + 0x00039e44, 0x00039e44, 0x00039e44, 0x00039e44, + 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, + 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, + 0x001bde2b, 0x001bde2b, 0x001bde2b, 0x001bde2b, + 0x001bde2b, 0x001bde2b, 0x001bde2b, 0x001bde2b, + 0x12613e2b, 0x12613e2b, 0x12613e2b, 0x12613e2b, + 0x12613e2b, 0x12613e2b, 0x12613e2b, 0x12613e2b, + 0x0023e92b, 0x0023e92b, 0x0023e92b, 0x0023e92b, + 0x0023e92b, 0x0023e92b, 0x0023e92b, 0x0023e92b, + 0x93c9492b, 0x93c9492b, 0x93c9492b, 0x93c9492b, + 0x93c9492b, 0x93c9492b, 0x93c9492b, 0x93c9492b, + 0x00299658, 0x00299658, 0x00299658, 0x00299658, + 0x00299658, 0x00299658, 0x00299658, 0x00299658, + 0x66f49658, 0x66f49658, 0x66f49658, 0x66f49658, + 0x66f49658, 0x66f49658, 0x66f49658, 0x66f49658, + 0x000fa070, 0x000fa070, 0x000fa070, 0x000fa070, + 0x000fa070, 0x000fa070, 0x000fa070, 0x000fa070, + 0x7c1da070, 0x7c1da070, 0x7c1da070, 0x7c1da070, + 0x7c1da070, 0x7c1da070, 0x7c1da070, 0x7c1da070, + 0xffef85a4, 0xffef85a4, 0xffef85a4, 0xffef85a4, + 0xffef85a4, 0xffef85a4, 0xffef85a4, 0xffef85a4, + 0xaea405a4, 0xaea405a4, 0xaea405a4, 0xaea405a4, + 0xaea405a4, 0xaea405a4, 0xaea405a4, 0xaea405a4, + 0x0036b788, 0x0036b788, 0x0036b788, 0x0036b788, + 0x0036b788, 0x0036b788, 0x0036b788, 0x0036b788, + 0x3327b788, 0x3327b788, 0x3327b788, 0x3327b788, + 0x3327b788, 0x3327b788, 0x3327b788, 0x3327b788, + 0x00294a67, 0x00294a67, 0x00294a67, 0x00294a67, + 0x00017620, 0x00017620, 0x00017620, 0x00017620, + 0x91f62a67, 0x91f62a67, 0x91f62a67, 0x91f62a67, + 0x9ec57620, 0x9ec57620, 0x9ec57620, 0x9ec57620, + 0x002ef4cd, 0x002ef4cd, 0x002ef4cd, 0x002ef4cd, + 0x0035dec5, 0x0035dec5, 0x0035dec5, 0x0035dec5, + 0xac4894cd, 0xac4894cd, 0xac4894cd, 0xac4894cd, + 0x6d8e7ec5, 0x6d8e7ec5, 0x6d8e7ec5, 0x6d8e7ec5, + 0xffc406e5, 0xffc406e5, 0xffe8ac81, 0xffe8ac81, + 0xffc7e1cf, 0xffc7e1cf, 0xffd19819, 0xffd19819, + 0xa220a6e5, 0xa220a6e5, 0xd8f8cc81, 0xd8f8cc81, + 0x5081c1cf, 0x5081c1cf, 0x8a54b819, 0x8a54b819, + 0xffe9d65d, 0xffe9d65d, 0x003509ee, 0x003509ee, + 0x002135c7, 0x002135c7, 0xffe7cfbb, 0xffe7cfbb, + 0x8035765d, 0x8035765d, 0x6272c9ee, 0x6272c9ee, + 0x5f5a15c7, 0x5f5a15c7, 0x085f2fbb, 0x085f2fbb, + 0xffe6a503, 0xffe6a503, 0xffe6a503, 0xffe6a503, + 0xffc9302c, 0xffc9302c, 0xffc9302c, 0xffc9302c, + 0x5f070503, 0x5f070503, 0x5f070503, 0x5f070503, + 0xbfceb02c, 0xbfceb02c, 0xbfceb02c, 0xbfceb02c, + 0xffd947d4, 0xffd947d4, 0xffd947d4, 0xffd947d4, + 0x003bbeaf, 0x003bbeaf, 0x003bbeaf, 0x003bbeaf, + 0x8ed3c7d4, 0x8ed3c7d4, 0x8ed3c7d4, 0x8ed3c7d4, + 0xdc919eaf, 0xdc919eaf, 0xdc919eaf, 0xdc919eaf, + 0xffeccf75, 0xffeccf75, 0x001d9772, 0x001d9772, + 0xffc1b072, 0xffc1b072, 0xfff0bcf6, 0xfff0bcf6, + 0xb35b6f75, 0xb35b6f75, 0xc20bd772, 0xc20bd772, + 0xc4cff072, 0xc4cff072, 0x748f7cf6, 0x748f7cf6, + 0xffcf5280, 0xffcf5280, 0xffcfd2ae, 0xffcfd2ae, + 0xffc890e0, 0xffc890e0, 0x0001efca, 0x0001efca, + 0xaa1f5280, 0xaa1f5280, 0x5b2592ae, 0x5b2592ae, + 0x21e490e0, 0x21e490e0, 0x80fb2fca, 0x80fb2fca, + 0x001fea93, 0x0033ff5a, 0x002358d4, 0x003a41f8, + 0xffccff72, 0x00223dfb, 0xffdaab9f, 0xffc9a422, + 0xfff24a93, 0x3b1f3f5a, 0x513dd8d4, 0x2c7941f8, + 0xaebb3f72, 0x36619dfb, 0x01ce8b9f, 0xab4de422, + 0x000412f5, 0x00252587, 0xffed24f0, 0x00359b5d, + 0xffca48a0, 0xffc6a2fc, 0xffedbb56, 0xffcf45de, + 0xdbe2b2f5, 0xfd560587, 0xec8b24f0, 0x79213b5d, + 0x78de48a0, 0x462622fc, 0x64587b56, 0x718b05de, + 0x000dbe5e, 0x001c5e1a, 0x000de0e6, 0x000c7f5a, + 0x00078f83, 0xffe7628a, 0xffff5704, 0xfff806fc, + 0x00d97e5e, 0xe6df9e1a, 0xe12aa0e6, 0x4af7bf5a, + 0x3c77ef83, 0xcf38a28a, 0x78dfd704, 0x72d786fc, + 0xfff60021, 0xffd05af6, 0x001f0084, 0x0030ef86, + 0xffc9b97d, 0xfff7fcd6, 0xfff44592, 0xffc921c2, + 0x337a2021, 0x682f1af6, 0xae2f8084, 0x7321af86, + 0x6c79597d, 0xec92bcd6, 0x07a68592, 0x4b0161c2, + 0xfff42118, 0xfff42118, 0xfff42118, 0xfff42118, + 0xfff42118, 0xfff42118, 0xfff42118, 0xfff42118, + 0x58172118, 0x58172118, 0x58172118, 0x58172118, + 0x58172118, 0x58172118, 0x58172118, 0x58172118, + 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, + 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, + 0xae1024ad, 0xae1024ad, 0xae1024ad, 0xae1024ad, + 0xae1024ad, 0xae1024ad, 0xae1024ad, 0xae1024ad, + 0xffe0147f, 0xffe0147f, 0xffe0147f, 0xffe0147f, + 0xffe0147f, 0xffe0147f, 0xffe0147f, 0xffe0147f, + 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, + 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, + 0xfff79d90, 0xfff79d90, 0xfff79d90, 0xfff79d90, + 0xfff79d90, 0xfff79d90, 0xfff79d90, 0xfff79d90, + 0x6ba99d90, 0x6ba99d90, 0x6ba99d90, 0x6ba99d90, + 0x6ba99d90, 0x6ba99d90, 0x6ba99d90, 0x6ba99d90, + 0xffeeeaa0, 0xffeeeaa0, 0xffeeeaa0, 0xffeeeaa0, + 0xffeeeaa0, 0xffeeeaa0, 0xffeeeaa0, 0xffeeeaa0, + 0x0d42eaa0, 0x0d42eaa0, 0x0d42eaa0, 0x0d42eaa0, + 0x0d42eaa0, 0x0d42eaa0, 0x0d42eaa0, 0x0d42eaa0, + 0x0027f968, 0x0027f968, 0x0027f968, 0x0027f968, + 0x0027f968, 0x0027f968, 0x0027f968, 0x0027f968, + 0xeb54f968, 0xeb54f968, 0xeb54f968, 0xeb54f968, + 0xeb54f968, 0xeb54f968, 0xeb54f968, 0xeb54f968, + 0xffdfd37b, 0xffdfd37b, 0xffdfd37b, 0xffdfd37b, + 0xffdfd37b, 0xffdfd37b, 0xffdfd37b, 0xffdfd37b, + 0x28cf337b, 0x28cf337b, 0x28cf337b, 0x28cf337b, + 0x28cf337b, 0x28cf337b, 0x28cf337b, 0x28cf337b, + 0xffc51585, 0xffc51585, 0xffc51585, 0xffc51585, + 0xffd18e7c, 0xffd18e7c, 0xffd18e7c, 0xffd18e7c, + 0xf3f5b585, 0xf3f5b585, 0xf3f5b585, 0xf3f5b585, + 0xe3a10e7c, 0xe3a10e7c, 0xe3a10e7c, 0xe3a10e7c, + 0x00368a96, 0x00368a96, 0x00368a96, 0x00368a96, + 0xffd43e41, 0xffd43e41, 0xffd43e41, 0xffd43e41, + 0xde894a96, 0xde894a96, 0xde894a96, 0xde894a96, + 0x6b1c5e41, 0x6b1c5e41, 0x6b1c5e41, 0x6b1c5e41, + 0x003410f2, 0x003410f2, 0xfff0fe85, 0xfff0fe85, + 0x0020c638, 0x0020c638, 0x00296e9f, 0x00296e9f, + 0xd15250f2, 0xd15250f2, 0xf1419e85, 0xf1419e85, + 0xdce7c638, 0xdce7c638, 0x5a7d4e9f, 0x5a7d4e9f, + 0xffd2b7a3, 0xffd2b7a3, 0xffc7a44b, 0xffc7a44b, + 0xfff9ba6d, 0xfff9ba6d, 0xffda3409, 0xffda3409, + 0x114717a3, 0x114717a3, 0xfad1044b, 0xfad1044b, + 0xb4c75a6d, 0xb4c75a6d, 0x65db5409, 0x65db5409, + 0x00360400, 0x00360400, 0x00360400, 0x00360400, + 0xfffb6a4d, 0xfffb6a4d, 0xfffb6a4d, 0xfffb6a4d, + 0xc0b60400, 0xc0b60400, 0xc0b60400, 0xc0b60400, + 0x7ac50a4d, 0x7ac50a4d, 0x7ac50a4d, 0x7ac50a4d, + 0x0023d69c, 0x0023d69c, 0x0023d69c, 0x0023d69c, + 0xfff7c55d, 0xfff7c55d, 0xfff7c55d, 0xfff7c55d, + 0x9cf7569c, 0x9cf7569c, 0x9cf7569c, 0x9cf7569c, + 0xbe23655d, 0xbe23655d, 0xbe23655d, 0xbe23655d, + 0xfff5c282, 0xfff5c282, 0xffed4113, 0xffed4113, + 0xffffa63b, 0xffffa63b, 0xffec09f7, 0xffec09f7, + 0x7f460282, 0x7f460282, 0x6a8fa113, 0x6a8fa113, + 0xc347063b, 0xc347063b, 0x61aae9f7, 0x61aae9f7, + 0xfffa2bdd, 0xfffa2bdd, 0x001495d4, 0x001495d4, + 0x001c4563, 0x001c4563, 0xffea2c62, 0xffea2c62, + 0xcaf5cbdd, 0xcaf5cbdd, 0xf8cf15d4, 0xf8cf15d4, + 0x6348a563, 0x6348a563, 0x9c766c62, 0x9c766c62, + 0x00053919, 0x0004610c, 0xffdacd41, 0x003eb01b, + 0x003472e7, 0xffcd003b, 0x001a7cc7, 0x00031924, + 0x7ea85919, 0x3625e10c, 0xbd02ed41, 0x34c2101b, + 0xb71152e7, 0x6e54603b, 0x08335cc7, 0x61279924, + 0x002b5ee5, 0x00291199, 0xffd87a3a, 0x00134d71, + 0x003de11c, 0x00130984, 0x0025f051, 0x00185a46, + 0x8d87fee5, 0xb9dc3199, 0xda1fba3a, 0x75416d71, + 0x9e61611c, 0xaf438984, 0xd9b01051, 0x00611a46, + 0xffc68518, 0x001314be, 0x00283891, 0xffc9db90, + 0xffd25089, 0x001c853f, 0x001d0b4b, 0xffeff6a6, + 0xa4698518, 0xfbaad4be, 0x02ba5891, 0xb33bdb90, + 0x29637089, 0xed44653f, 0x28066b4b, 0x43c4b6a6, + 0xffeba8be, 0x0012e11b, 0xffcd5e3e, 0xffea2d2f, + 0xfff91de4, 0x001406c7, 0x00327283, 0xffe20d6e, + 0x0e0368be, 0x3ab6411b, 0x84951e3e, 0x6a100d2f, + 0xc1b59de4, 0x396ce6c7, 0x1902d283, 0x428fcd6e, + 0xfff2a128, 0xfff2a128, 0xfff2a128, 0xfff2a128, + 0xfff2a128, 0xfff2a128, 0xfff2a128, 0xfff2a128, + 0x6017a128, 0x6017a128, 0x6017a128, 0x6017a128, + 0x6017a128, 0x6017a128, 0x6017a128, 0x6017a128, + 0x002f9a75, 0x002f9a75, 0x002f9a75, 0x002f9a75, + 0x002f9a75, 0x002f9a75, 0x002f9a75, 0x002f9a75, + 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, + 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, + 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, + 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, + 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, + 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, + 0xffdfadd6, 0xffdfadd6, 0xffdfadd6, 0xffdfadd6, + 0xffdfadd6, 0xffdfadd6, 0xffdfadd6, 0xffdfadd6, + 0x629a6dd6, 0x629a6dd6, 0x629a6dd6, 0x629a6dd6, + 0x629a6dd6, 0x629a6dd6, 0x629a6dd6, 0x629a6dd6, + 0xffc51ae7, 0xffc51ae7, 0xffc51ae7, 0xffc51ae7, + 0xffc51ae7, 0xffc51ae7, 0xffc51ae7, 0xffc51ae7, + 0xcba1fae7, 0xcba1fae7, 0xcba1fae7, 0xcba1fae7, + 0xcba1fae7, 0xcba1fae7, 0xcba1fae7, 0xcba1fae7, + 0xffeaa4f7, 0xffeaa4f7, 0xffeaa4f7, 0xffeaa4f7, + 0xffeaa4f7, 0xffeaa4f7, 0xffeaa4f7, 0xffeaa4f7, + 0xb50984f7, 0xb50984f7, 0xb50984f7, 0xb50984f7, + 0xb50984f7, 0xb50984f7, 0xb50984f7, 0xb50984f7, + 0xffcdfc98, 0xffcdfc98, 0xffcdfc98, 0xffcdfc98, + 0xffcdfc98, 0xffcdfc98, 0xffcdfc98, 0xffcdfc98, + 0xd360fc98, 0xd360fc98, 0xd360fc98, 0xd360fc98, + 0xd360fc98, 0xd360fc98, 0xd360fc98, 0xd360fc98, + 0xffe6123d, 0xffe6123d, 0xffe6123d, 0xffe6123d, + 0xffe6ead6, 0xffe6ead6, 0xffe6ead6, 0xffe6ead6, + 0x97adb23d, 0x97adb23d, 0x97adb23d, 0x97adb23d, + 0xca41aad6, 0xca41aad6, 0xca41aad6, 0xca41aad6, + 0x00357e1e, 0x00357e1e, 0x00357e1e, 0x00357e1e, + 0xffc5af59, 0xffc5af59, 0xffc5af59, 0xffc5af59, + 0x18f93e1e, 0x18f93e1e, 0x18f93e1e, 0x18f93e1e, + 0x6d30cf59, 0x6d30cf59, 0x6d30cf59, 0x6d30cf59, + 0xffccfbe9, 0xffccfbe9, 0x00040af0, 0x00040af0, + 0x0007c417, 0x0007c417, 0x002f4588, 0x002f4588, + 0x4eca1be9, 0x4eca1be9, 0xc9620af0, 0xc9620af0, + 0x490aa417, 0x490aa417, 0x44e04588, 0x44e04588, + 0x0000ad00, 0x0000ad00, 0xffef36be, 0xffef36be, + 0x000dcd44, 0x000dcd44, 0x003c675a, 0x003c675a, + 0x95a0ad00, 0x95a0ad00, 0x7fc6f6be, 0x7fc6f6be, + 0x27b64d44, 0x27b64d44, 0x4827a75a, 0x4827a75a, + 0x0035843f, 0x0035843f, 0x0035843f, 0x0035843f, + 0xffdf5617, 0xffdf5617, 0xffdf5617, 0xffdf5617, + 0x8d3d643f, 0x8d3d643f, 0x8d3d643f, 0x8d3d643f, + 0x3b223617, 0x3b223617, 0x3b223617, 0x3b223617, + 0xffe7945c, 0xffe7945c, 0xffe7945c, 0xffe7945c, + 0x0038738c, 0x0038738c, 0x0038738c, 0x0038738c, + 0x3473145c, 0x3473145c, 0x3473145c, 0x3473145c, + 0x78a9f38c, 0x78a9f38c, 0x78a9f38c, 0x78a9f38c, + 0xffc72bca, 0xffc72bca, 0xffffde7e, 0xffffde7e, + 0x00193948, 0x00193948, 0xffce69c0, 0xffce69c0, + 0x28406bca, 0x28406bca, 0xb4cf9e7e, 0xb4cf9e7e, + 0xa3423948, 0xa3423948, 0xed0669c0, 0xed0669c0, + 0x0024756c, 0x0024756c, 0xfffcc7df, 0xfffcc7df, + 0x000b98a1, 0x000b98a1, 0xffebe808, 0xffebe808, + 0x88d1f56c, 0x88d1f56c, 0x2578a7df, 0x2578a7df, + 0xa69fb8a1, 0xa69fb8a1, 0x98ece808, 0x98ece808, + 0xffec7953, 0x001d4099, 0xffd92578, 0xffeb05ad, + 0x0016e405, 0x000bdbe7, 0x00221de8, 0x0033f8cf, + 0x3196d953, 0xbfb06099, 0x48882578, 0x3e20a5ad, + 0xee178405, 0x2408bbe7, 0xefdf1de8, 0x53cdd8cf, + 0xfff7b934, 0xffd4ca0c, 0xffe67ff8, 0xffe3d157, + 0xffd8911b, 0xffc72c12, 0x000910d8, 0xffc65e1f, + 0x2d1e3934, 0xc3164a0c, 0xb3e57ff8, 0x2a8eb157, + 0xf07bf11b, 0x24496c12, 0x162410d8, 0x380a3e1f, + 0xffe14658, 0x00251d8b, 0x002573b7, 0xfffd7c8f, + 0x001ddd98, 0x00336898, 0x0002d4bb, 0xffed93a7, + 0x5cac4658, 0x0a567d8b, 0xaf1c53b7, 0xa40f5c8f, + 0x4fd0dd98, 0x81466898, 0xe91a34bb, 0x7ae273a7, + 0xffcf6cbe, 0x00027c1c, 0x0018aa08, 0x002dfd71, + 0x000c5ca5, 0x0019379a, 0xffc7a167, 0xffe48c3d, + 0x86672cbe, 0xb185fc1c, 0x3159aa08, 0xcb5c1d71, + 0xcd20fca5, 0xc20c779a, 0xdc748167, 0x66ec2c3d, + 0x00071e24, 0x00071e24, 0x00071e24, 0x00071e24, + 0x00071e24, 0x00071e24, 0x00071e24, 0x00071e24, + 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, + 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, + 0x002f7a49, 0x002f7a49, 0x002f7a49, 0x002f7a49, + 0x002f7a49, 0x002f7a49, 0x002f7a49, 0x002f7a49, + 0xeef89a49, 0xeef89a49, 0xeef89a49, 0xeef89a49, + 0xeef89a49, 0xeef89a49, 0xeef89a49, 0xeef89a49, + 0x0028e527, 0x0028e527, 0x0028e527, 0x0028e527, + 0x0028e527, 0x0028e527, 0x0028e527, 0x0028e527, + 0x254dc527, 0x254dc527, 0x254dc527, 0x254dc527, + 0x254dc527, 0x254dc527, 0x254dc527, 0x254dc527, + 0x001ad035, 0x001ad035, 0x001ad035, 0x001ad035, + 0x001ad035, 0x001ad035, 0x001ad035, 0x001ad035, + 0x13a17035, 0x13a17035, 0x13a17035, 0x13a17035, + 0x13a17035, 0x13a17035, 0x13a17035, 0x13a17035, + 0xffffb422, 0xffffb422, 0xffffb422, 0xffffb422, + 0xffffb422, 0xffffb422, 0xffffb422, 0xffffb422, + 0x6d83f422, 0x6d83f422, 0x6d83f422, 0x6d83f422, + 0x6d83f422, 0x6d83f422, 0x6d83f422, 0x6d83f422, + 0x003d3201, 0x003d3201, 0x003d3201, 0x003d3201, + 0x003d3201, 0x003d3201, 0x003d3201, 0x003d3201, + 0xa9fd5201, 0xa9fd5201, 0xa9fd5201, 0xa9fd5201, + 0xa9fd5201, 0xa9fd5201, 0xa9fd5201, 0xa9fd5201, + 0x000445c5, 0x000445c5, 0x000445c5, 0x000445c5, + 0x000445c5, 0x000445c5, 0x000445c5, 0x000445c5, + 0xba3ce5c5, 0xba3ce5c5, 0xba3ce5c5, 0xba3ce5c5, + 0xba3ce5c5, 0xba3ce5c5, 0xba3ce5c5, 0xba3ce5c5, + 0x000c63a8, 0x000c63a8, 0x000c63a8, 0x000c63a8, + 0x00081b9a, 0x00081b9a, 0x00081b9a, 0x00081b9a, + 0x588163a8, 0x588163a8, 0x588163a8, 0x588163a8, + 0x9e7b5b9a, 0x9e7b5b9a, 0x9e7b5b9a, 0x9e7b5b9a, + 0x000e8f76, 0x000e8f76, 0x000e8f76, 0x000e8f76, + 0x003b3853, 0x003b3853, 0x003b3853, 0x003b3853, + 0xeefd4f76, 0xeefd4f76, 0xeefd4f76, 0xeefd4f76, + 0x89c59853, 0x89c59853, 0x89c59853, 0x89c59853, + 0x0002e46c, 0x0002e46c, 0xffc9c808, 0xffc9c808, + 0x003036c2, 0x003036c2, 0xffe3bff6, 0xffe3bff6, + 0xd690646c, 0xd690646c, 0x54cac808, 0x54cac808, + 0xae0876c2, 0xae0876c2, 0x54e27ff6, 0x54e27ff6, + 0xffdb3c93, 0xffdb3c93, 0xfffd4ae0, 0xfffd4ae0, + 0x00141305, 0x00141305, 0x00147792, 0x00147792, + 0x69ed9c93, 0x69ed9c93, 0xb9594ae0, 0xb9594ae0, + 0x13f4b305, 0x13f4b305, 0x0e06b792, 0x0e06b792, + 0x003b8534, 0x003b8534, 0x003b8534, 0x003b8534, + 0xffd8fc30, 0xffd8fc30, 0xffd8fc30, 0xffd8fc30, + 0xa6e20534, 0xa6e20534, 0xa6e20534, 0xa6e20534, + 0xc75efc30, 0xc75efc30, 0xc75efc30, 0xc75efc30, + 0x001f9d54, 0x001f9d54, 0x001f9d54, 0x001f9d54, + 0xffd54f2d, 0xffd54f2d, 0xffd54f2d, 0xffd54f2d, + 0x99ca1d54, 0x99ca1d54, 0x99ca1d54, 0x99ca1d54, + 0xc73aef2d, 0xc73aef2d, 0xc73aef2d, 0xc73aef2d, + 0x00139e25, 0x00139e25, 0xffe7d0e0, 0xffe7d0e0, + 0xfff39944, 0xfff39944, 0xffea0802, 0xffea0802, + 0xf5583e25, 0xf5583e25, 0x0a03d0e0, 0x0a03d0e0, + 0xe11c1944, 0xe11c1944, 0x47ea4802, 0x47ea4802, + 0xffd1eea2, 0xffd1eea2, 0xffc4c79c, 0xffc4c79c, + 0xffc8a057, 0xffc8a057, 0x003a97d9, 0x003a97d9, + 0x74a62ea2, 0x74a62ea2, 0x3ab8479c, 0x3ab8479c, + 0x44538057, 0x44538057, 0xcab5b7d9, 0xcab5b7d9, + 0xffd1a13c, 0x0035c539, 0x003b0115, 0x00041dc0, + 0x0021c4f7, 0xfff11bf4, 0x001a35e7, 0x0007340e, + 0x85f9213c, 0x005ce539, 0x29dda115, 0xa3bc1dc0, + 0x9940a4f7, 0xf96f9bf4, 0xef5715e7, 0x1788f40e, + 0xfff97d45, 0x001a4cd0, 0xffe47cae, 0x001d2668, + 0xffe68e98, 0xffef2633, 0xfffc05da, 0xffc57fdb, + 0xa1221d45, 0x21b44cd0, 0xf07a3cae, 0x10ea2668, + 0xe5b98e98, 0x97358633, 0xfbb745da, 0x2e40dfdb, + 0xffd32764, 0xffdde1af, 0xfff993dd, 0xffdd1d09, + 0x0002cc93, 0xfff11805, 0x00189c2a, 0xffc9e5a9, + 0x42bfa764, 0xa093c1af, 0xb7f533dd, 0x42fe3d09, + 0x5c152c93, 0x3471b805, 0xa69ddc2a, 0x0bff05a9, + 0xfff78a50, 0x003bcf2c, 0xffff434e, 0xffeb36df, + 0x003c15ca, 0x00155e68, 0xfff316b6, 0x001e29ce, + 0x09418a50, 0x94214f2c, 0x7969034e, 0x734716df, + 0xc5f555ca, 0x17e25e68, 0xdfc9d6b6, 0x1657e9ce, +}; + +XALIGNED(16) static const word32 L_mldsa_avx2_zeta1[] WC_X64I_UNUSED = { + 0xffc97e01, 0xffc97e01, 0xffc97e01, 0xffc97e01, + 0xffc97e01, 0xffc97e01, 0xffc97e01, 0xffc97e01, +}; + +XALIGNED(16) static const word32 L_mldsa_avx2_half[] WC_X64I_UNUSED = { + 0x00400000, 0x00400000, 0x00400000, 0x00400000, + 0x00400000, 0x00400000, 0x00400000, 0x00400000, +}; + +XALIGNED(16) static const word32 L_mldsa_avx2_zetas_basemul[] WC_X64I_UNUSED = { + 0xffc406e5, 0xffe9d65d, 0x003cf91b, 0x001729a3, + 0xffe8ac81, 0x003509ee, 0x0018537f, 0xffcbf612, + 0xffc7e1cf, 0x002135c7, 0x00391e31, 0xffdfca39, + 0xffd19819, 0xffe7cfbb, 0x002f67e7, 0x00193045, + 0x0000a6e5, 0x0000765d, 0x0000591b, 0x000089a3, + 0x0000cc81, 0x0000c9ee, 0x0000337f, 0x00003612, + 0x0000c1cf, 0x000015c7, 0x00003e31, 0x0000ea39, + 0x0000b819, 0x00002fbb, 0x000047e7, 0x0000d045, + 0xffeccf75, 0xffcf5280, 0x0014308b, 0x0031ad80, + 0x001d9772, 0xffcfd2ae, 0xffe3688e, 0x00312d52, + 0xffc1b072, 0xffc890e0, 0x003f4f8e, 0x00386f20, + 0xfff0bcf6, 0x0001efca, 0x0010430a, 0xffff1036, + 0x00006f75, 0x00005280, 0x0000908b, 0x0000ad80, + 0x0000d772, 0x000092ae, 0x0000288e, 0x00006d52, + 0x0000f072, 0x000090e0, 0x00000f8e, 0x00006f20, + 0x00007cf6, 0x00002fca, 0x0000830a, 0x0000d036, + 0x003410f2, 0xffd2b7a3, 0xffccef0e, 0x002e485d, + 0xfff0fe85, 0xffc7a44b, 0x0010017b, 0x00395bb5, + 0x0020c638, 0xfff9ba6d, 0xffe039c8, 0x00074593, + 0x00296e9f, 0xffda3409, 0xffd79161, 0x0026cbf7, + 0x000050f2, 0x000017a3, 0x0000af0e, 0x0000e85d, + 0x00009e85, 0x0000044b, 0x0000617b, 0x0000fbb5, + 0x0000c638, 0x00005a6d, 0x000039c8, 0x0000a593, + 0x00004e9f, 0x00005409, 0x0000b161, 0x0000abf7, + 0xfff5c282, 0xfffa2bdd, 0x000b3d7e, 0x0006d423, + 0xffed4113, 0x001495d4, 0x0013beed, 0xffec6a2c, + 0xffffa63b, 0x001c4563, 0x000159c5, 0xffe4ba9d, + 0xffec09f7, 0xffea2c62, 0x0014f609, 0x0016d39e, + 0x00000282, 0x0000cbdd, 0x0000fd7e, 0x00003423, + 0x0000a113, 0x000015d4, 0x00005eed, 0x0000ea2c, + 0x0000063b, 0x0000a563, 0x0000f9c5, 0x00005a9d, + 0x0000e9f7, 0x00006c62, 0x00001609, 0x0000939e, + 0xffccfbe9, 0x0000ad00, 0x00340417, 0x00005300, + 0x00040af0, 0xffef36be, 0xfffcf510, 0x0011c942, + 0x0007c417, 0x000dcd44, 0xfff93be9, 0xfff332bc, + 0x002f4588, 0x003c675a, 0xffd1ba78, 0xffc498a6, + 0x00001be9, 0x0000ad00, 0x0000e417, 0x00005300, + 0x00000af0, 0x0000f6be, 0x0000f510, 0x00000942, + 0x0000a417, 0x00004d44, 0x00005be9, 0x0000b2bc, + 0x00004588, 0x0000a75a, 0x0000ba78, 0x000058a6, + 0xffc72bca, 0x0024756c, 0x0039d436, 0xffdc8a94, + 0xffffde7e, 0xfffcc7df, 0x00012182, 0x00043821, + 0x00193948, 0x000b98a1, 0xffe7c6b8, 0xfff5675f, + 0xffce69c0, 0xffebe808, 0x00329640, 0x001517f8, + 0x00006bca, 0x0000f56c, 0x00009436, 0x00000a94, + 0x00009e7e, 0x0000a7df, 0x00006182, 0x00005821, + 0x00003948, 0x0000b8a1, 0x0000c6b8, 0x0000475f, + 0x000069c0, 0x0000e808, 0x00009640, 0x000017f8, + 0x0002e46c, 0xffdb3c93, 0xfffe1b94, 0x0025c36d, + 0xffc9c808, 0xfffd4ae0, 0x003737f8, 0x0003b520, + 0x003036c2, 0x00141305, 0xffd0c93e, 0xffececfb, + 0xffe3bff6, 0x00147792, 0x001d400a, 0xffec886e, + 0x0000646c, 0x00009c93, 0x00009b94, 0x0000636d, + 0x0000c808, 0x00004ae0, 0x000037f8, 0x0000b520, + 0x000076c2, 0x0000b305, 0x0000893e, 0x00004cfb, + 0x00007ff6, 0x0000b792, 0x0000800a, 0x0000486e, + 0x00139e25, 0xffd1eea2, 0xffed61db, 0x002f115e, + 0xffe7d0e0, 0xffc4c79c, 0x00192f20, 0x003c3864, + 0xfff39944, 0xffc8a057, 0x000d66bc, 0x00385fa9, + 0xffea0802, 0x003a97d9, 0x0016f7fe, 0xffc66827, + 0x00003e25, 0x00002ea2, 0x0000c1db, 0x0000d15e, + 0x0000d0e0, 0x0000479c, 0x00002f20, 0x0000b864, + 0x00001944, 0x00008057, 0x0000e6bc, 0x00007fa9, + 0x00004802, 0x0000b7d9, 0x0000b7fe, 0x00004827, +}; + +XALIGNED(16) static const word32 L_mldsa_avx2_zetas_1[] WC_X64I_UNUSED = { + 0xffc97e01, 0xffc97e01, 0xffc97e01, 0xffc97e01, + 0xffc97e01, 0xffc97e01, 0xffc97e01, 0xffc97e01, +}; + +XALIGNED(16) static const word32 L_mldsa_avx2_zetas_inv[] WC_X64I_UNUSED = { + 0xffe1d632, 0x000ce94a, 0xffeaa198, 0xffc3ea36, + 0x0014c921, 0x0000bcb2, 0xffc430d4, 0x000875b0, + 0xe9a81632, 0x2036294a, 0xe81da198, 0x3a0aaa36, + 0x8cb8e921, 0x8696fcb2, 0x6bdeb0d4, 0xf6be75b0, + 0x00361a57, 0xffe763d6, 0x000ee7fb, 0xfffd336d, + 0x0022e2f7, 0x00066c23, 0x00221e51, 0x002cd89c, + 0xf400fa57, 0x596223d6, 0xcb8e47fb, 0xa3ead36d, + 0xbd01c2f7, 0x480acc23, 0x5f6c3e51, 0xbd40589c, + 0xffc56827, 0xffc56827, 0x00375fa9, 0x00375fa9, + 0x003b3864, 0x003b3864, 0x002e115e, 0x002e115e, + 0x354a4827, 0x354a4827, 0xbbac7fa9, 0xbbac7fa9, + 0xc547b864, 0xc547b864, 0x8b59d15e, 0x8b59d15e, + 0x0015f7fe, 0x0015f7fe, 0x000c66bc, 0x000c66bc, + 0x00182f20, 0x00182f20, 0xffec61db, 0xffec61db, + 0xb815b7fe, 0xb815b7fe, 0x1ee3e6bc, 0x1ee3e6bc, + 0xf5fc2f20, 0xf5fc2f20, 0x0aa7c1db, 0x0aa7c1db, + 0x002ab0d3, 0x002ab0d3, 0x002ab0d3, 0x002ab0d3, + 0xffe062ac, 0xffe062ac, 0xffe062ac, 0xffe062ac, + 0x38c510d3, 0x38c510d3, 0x38c510d3, 0x38c510d3, + 0x6635e2ac, 0x6635e2ac, 0x6635e2ac, 0x6635e2ac, + 0x002703d0, 0x002703d0, 0x002703d0, 0x002703d0, + 0xffc47acc, 0xffc47acc, 0xffc47acc, 0xffc47acc, + 0x38a103d0, 0x38a103d0, 0x38a103d0, 0x38a103d0, + 0x591dfacc, 0x591dfacc, 0x591dfacc, 0x591dfacc, + 0xfffbba3b, 0xfffbba3b, 0xfffbba3b, 0xfffbba3b, + 0xfffbba3b, 0xfffbba3b, 0xfffbba3b, 0xfffbba3b, + 0x45c31a3b, 0x45c31a3b, 0x45c31a3b, 0x45c31a3b, + 0x45c31a3b, 0x45c31a3b, 0x45c31a3b, 0x45c31a3b, + 0xffc2cdff, 0xffc2cdff, 0xffc2cdff, 0xffc2cdff, + 0xffc2cdff, 0xffc2cdff, 0xffc2cdff, 0xffc2cdff, + 0x5602adff, 0x5602adff, 0x5602adff, 0x5602adff, + 0x5602adff, 0x5602adff, 0x5602adff, 0x5602adff, + 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, + 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, + 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, + 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, + 0x003a8025, 0x0003fa26, 0x0010d9cd, 0x00197168, + 0xffe2d998, 0x001b8352, 0xffe5b330, 0x000682bb, + 0xd1bf2025, 0x0448ba26, 0x68ca79cd, 0x1a467168, + 0xef15d998, 0x0f85c352, 0xde4bb330, 0x5edde2bb, + 0xfff8cbf2, 0xffe5ca19, 0x000ee40c, 0xffde3b09, + 0xfffbe240, 0xffc4feeb, 0xffca3ac7, 0x002e5ec4, + 0xe8770bf2, 0x10a8ea19, 0x0690640c, 0x66bf5b09, + 0x5c43e240, 0xd6225eeb, 0xffa31ac7, 0x7a06dec4, + 0xffeb886e, 0xffeb886e, 0xffebecfb, 0xffebecfb, + 0x0002b520, 0x0002b520, 0x0024c36d, 0x0024c36d, + 0xf1f9486e, 0xf1f9486e, 0xec0b4cfb, 0xec0b4cfb, + 0x46a6b520, 0x46a6b520, 0x9612636d, 0x9612636d, + 0x001c400a, 0x001c400a, 0xffcfc93e, 0xffcfc93e, + 0x003637f8, 0x003637f8, 0xfffd1b94, 0xfffd1b94, + 0xab1d800a, 0xab1d800a, 0x51f7893e, 0x51f7893e, + 0xab3537f8, 0xab3537f8, 0x296f9b94, 0x296f9b94, + 0xffc4c7ad, 0xffc4c7ad, 0xffc4c7ad, 0xffc4c7ad, + 0xfff1708a, 0xfff1708a, 0xfff1708a, 0xfff1708a, + 0x763a67ad, 0x763a67ad, 0x763a67ad, 0x763a67ad, + 0x1102b08a, 0x1102b08a, 0x1102b08a, 0x1102b08a, + 0xfff7e466, 0xfff7e466, 0xfff7e466, 0xfff7e466, + 0xfff39c58, 0xfff39c58, 0xfff39c58, 0xfff39c58, + 0x6184a466, 0x6184a466, 0x6184a466, 0x6184a466, + 0xa77e9c58, 0xa77e9c58, 0xa77e9c58, 0xa77e9c58, + 0x00004bde, 0x00004bde, 0x00004bde, 0x00004bde, + 0x00004bde, 0x00004bde, 0x00004bde, 0x00004bde, + 0x927c0bde, 0x927c0bde, 0x927c0bde, 0x927c0bde, + 0x927c0bde, 0x927c0bde, 0x927c0bde, 0x927c0bde, + 0xffe52fcb, 0xffe52fcb, 0xffe52fcb, 0xffe52fcb, + 0xffe52fcb, 0xffe52fcb, 0xffe52fcb, 0xffe52fcb, + 0xec5e8fcb, 0xec5e8fcb, 0xec5e8fcb, 0xec5e8fcb, + 0xec5e8fcb, 0xec5e8fcb, 0xec5e8fcb, 0xec5e8fcb, + 0xffd085b7, 0xffd085b7, 0xffd085b7, 0xffd085b7, + 0xffd085b7, 0xffd085b7, 0xffd085b7, 0xffd085b7, + 0x110765b7, 0x110765b7, 0x110765b7, 0x110765b7, + 0x110765b7, 0x110765b7, 0x110765b7, 0x110765b7, + 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, + 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, + 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, + 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, + 0x001b73c3, 0x00385e99, 0xffe6c866, 0xfff3a35b, + 0xffd2028f, 0xffe755f8, 0xfffd83e4, 0x00309342, + 0x9913d3c3, 0x238b7e99, 0x3df38866, 0x32df035b, + 0x34a3e28f, 0xcea655f8, 0x4e7a03e4, 0x7998d342, + 0x00126c59, 0xfffd2b45, 0xffcc9768, 0xffe22268, + 0x00028371, 0xffda8c49, 0xffdae275, 0x001eb9a8, + 0x851d8c59, 0x16e5cb45, 0x7eb99768, 0xb02f2268, + 0x5bf0a371, 0x50e3ac49, 0xf5a98275, 0xa353b9a8, + 0x001417f8, 0x001417f8, 0xfff4675f, 0xfff4675f, + 0x00033821, 0x00033821, 0xffdb8a94, 0xffdb8a94, + 0x671317f8, 0x671317f8, 0x5960475f, 0x5960475f, + 0xda875821, 0xda875821, 0x772e0a94, 0x772e0a94, + 0x00319640, 0x00319640, 0xffe6c6b8, 0xffe6c6b8, + 0x00002182, 0x00002182, 0x0038d436, 0x0038d436, + 0x12f99640, 0x12f99640, 0x5cbdc6b8, 0x5cbdc6b8, + 0x4b306182, 0x4b306182, 0xd7bf9436, 0xd7bf9436, + 0xffc78c74, 0xffc78c74, 0xffc78c74, 0xffc78c74, + 0x00186ba4, 0x00186ba4, 0x00186ba4, 0x00186ba4, + 0x87560c74, 0x87560c74, 0x87560c74, 0x87560c74, + 0xcb8ceba4, 0xcb8ceba4, 0xcb8ceba4, 0xcb8ceba4, + 0x0020a9e9, 0x0020a9e9, 0x0020a9e9, 0x0020a9e9, + 0xffca7bc1, 0xffca7bc1, 0xffca7bc1, 0xffca7bc1, + 0xc4ddc9e9, 0xc4ddc9e9, 0xc4ddc9e9, 0xc4ddc9e9, + 0x72c29bc1, 0x72c29bc1, 0x72c29bc1, 0x72c29bc1, + 0x00320368, 0x00320368, 0x00320368, 0x00320368, + 0x00320368, 0x00320368, 0x00320368, 0x00320368, + 0x2c9f0368, 0x2c9f0368, 0x2c9f0368, 0x2c9f0368, + 0x2c9f0368, 0x2c9f0368, 0x2c9f0368, 0x2c9f0368, + 0x00155b09, 0x00155b09, 0x00155b09, 0x00155b09, + 0x00155b09, 0x00155b09, 0x00155b09, 0x00155b09, + 0x4af67b09, 0x4af67b09, 0x4af67b09, 0x4af67b09, + 0x4af67b09, 0x4af67b09, 0x4af67b09, 0x4af67b09, + 0x002c04f7, 0x002c04f7, 0x002c04f7, 0x002c04f7, + 0x002c04f7, 0x002c04f7, 0x002c04f7, 0x002c04f7, + 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, + 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, + 0x0039a1e1, 0xfff6ef28, 0x0038d3ee, 0x00276ee5, + 0x001c2ea9, 0x00198008, 0x002b35f4, 0x000846cc, + 0xc7f5c1e1, 0xe9dbef28, 0xdbb693ee, 0x0f840ee5, + 0xd5714ea9, 0x4c1a8008, 0x3ce9b5f4, 0xd2e1c6cc, + 0xffcc0731, 0xffdde218, 0xfff42419, 0xffe91bfb, + 0x0014fa53, 0x0026da88, 0xffe2bf67, 0x001386ad, + 0xac322731, 0x1020e218, 0xdbf74419, 0x11e87bfb, + 0xc1df5a53, 0xb777da88, 0x404f9f67, 0xce6926ad, + 0xffc398a6, 0xffc398a6, 0xfff232bc, 0xfff232bc, + 0x0010c942, 0x0010c942, 0xffff5300, 0xffff5300, + 0xb7d858a6, 0xb7d858a6, 0xd849b2bc, 0xd849b2bc, + 0x80390942, 0x80390942, 0x6a5f5300, 0x6a5f5300, + 0xffd0ba78, 0xffd0ba78, 0xfff83be9, 0xfff83be9, + 0xfffbf510, 0xfffbf510, 0x00330417, 0x00330417, + 0xbb1fba78, 0xbb1fba78, 0xb6f55be9, 0xb6f55be9, + 0x369df510, 0x369df510, 0xb135e417, 0xb135e417, + 0x003a50a7, 0x003a50a7, 0x003a50a7, 0x003a50a7, + 0xffca81e2, 0xffca81e2, 0xffca81e2, 0xffca81e2, + 0x92cf30a7, 0x92cf30a7, 0x92cf30a7, 0x92cf30a7, + 0xe706c1e2, 0xe706c1e2, 0xe706c1e2, 0xe706c1e2, + 0x0019152a, 0x0019152a, 0x0019152a, 0x0019152a, + 0x0019edc3, 0x0019edc3, 0x0019edc3, 0x0019edc3, + 0x35be552a, 0x35be552a, 0x35be552a, 0x35be552a, + 0x68524dc3, 0x68524dc3, 0x68524dc3, 0x68524dc3, + 0x003ae519, 0x003ae519, 0x003ae519, 0x003ae519, + 0x003ae519, 0x003ae519, 0x003ae519, 0x003ae519, + 0x345e0519, 0x345e0519, 0x345e0519, 0x345e0519, + 0x345e0519, 0x345e0519, 0x345e0519, 0x345e0519, + 0x0020522a, 0x0020522a, 0x0020522a, 0x0020522a, + 0x0020522a, 0x0020522a, 0x0020522a, 0x0020522a, + 0x9d65922a, 0x9d65922a, 0x9d65922a, 0x9d65922a, + 0x9d65922a, 0x9d65922a, 0x9d65922a, 0x9d65922a, + 0xffd0658b, 0xffd0658b, 0xffd0658b, 0xffd0658b, + 0xffd0658b, 0xffd0658b, 0xffd0658b, 0xffd0658b, + 0x7301c58b, 0x7301c58b, 0x7301c58b, 0x7301c58b, + 0x7301c58b, 0x7301c58b, 0x7301c58b, 0x7301c58b, + 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, + 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, + 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, + 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, + 0x001df292, 0xffcd8d7d, 0xffebf939, 0x0006e21c, + 0x0015d2d1, 0x0032a1c2, 0xffed1ee5, 0x00145742, + 0xbd703292, 0xe6fd2d7d, 0xc6931939, 0x3e4a621c, + 0x95eff2d1, 0x7b6ae1c2, 0xc549bee5, 0xf1fc9742, + 0x0010095a, 0xffe2f4b5, 0xffe37ac1, 0x002daf77, + 0x00362470, 0xffd7c76f, 0xffeceb42, 0x00397ae8, + 0xbc3b495a, 0xd7f994b5, 0x12bb9ac1, 0xd69c8f77, + 0x4cc42470, 0xfd45a76f, 0x04552b42, 0x5b967ae8, + 0x0015d39e, 0x0015d39e, 0xffe3ba9d, 0xffe3ba9d, + 0xffeb6a2c, 0xffeb6a2c, 0x0005d423, 0x0005d423, + 0x6389939e, 0x6389939e, 0x9cb75a9d, 0x9cb75a9d, + 0x0730ea2c, 0x0730ea2c, 0x350a3423, 0x350a3423, + 0x0013f609, 0x0013f609, 0x000059c5, 0x000059c5, + 0x0012beed, 0x0012beed, 0x000a3d7e, 0x000a3d7e, + 0x9e551609, 0x9e551609, 0x3cb8f9c5, 0x3cb8f9c5, + 0x95705eed, 0x95705eed, 0x80b9fd7e, 0x80b9fd7e, + 0x00083aa3, 0x00083aa3, 0x00083aa3, 0x00083aa3, + 0xffdc2964, 0xffdc2964, 0xffdc2964, 0xffdc2964, + 0x41dc9aa3, 0x41dc9aa3, 0x41dc9aa3, 0x41dc9aa3, + 0x6308a964, 0x6308a964, 0x6308a964, 0x6308a964, + 0x000495b3, 0x000495b3, 0x000495b3, 0x000495b3, + 0xffc9fc00, 0xffc9fc00, 0xffc9fc00, 0xffc9fc00, + 0x853af5b3, 0x853af5b3, 0x853af5b3, 0x853af5b3, + 0x3f49fc00, 0x3f49fc00, 0x3f49fc00, 0x3f49fc00, + 0x00202c85, 0x00202c85, 0x00202c85, 0x00202c85, + 0x00202c85, 0x00202c85, 0x00202c85, 0x00202c85, + 0xd730cc85, 0xd730cc85, 0xd730cc85, 0xd730cc85, + 0xd730cc85, 0xd730cc85, 0xd730cc85, 0xd730cc85, + 0xffd80698, 0xffd80698, 0xffd80698, 0xffd80698, + 0xffd80698, 0xffd80698, 0xffd80698, 0xffd80698, + 0x14ab0698, 0x14ab0698, 0x14ab0698, 0x14ab0698, + 0x14ab0698, 0x14ab0698, 0x14ab0698, 0x14ab0698, + 0x001feb81, 0x001feb81, 0x001feb81, 0x001feb81, + 0x001feb81, 0x001feb81, 0x001feb81, 0x001feb81, + 0x41100b81, 0x41100b81, 0x41100b81, 0x41100b81, + 0x41100b81, 0x41100b81, 0x41100b81, 0x41100b81, + 0xffe7a5ba, 0xffda0faf, 0xffecf67c, 0xffc21ee4, + 0xffecb28f, 0x002785c6, 0xffd6ee67, 0xffd4a11b, + 0xff9ee5ba, 0x264fefaf, 0x50bc767c, 0x619e9ee4, + 0x8abe928f, 0x25e045c6, 0x4623ce67, 0x7278011b, + 0xfffce6dc, 0xffe58339, 0x0032ffc5, 0xffcb8d19, + 0xffc14fe5, 0x002532bf, 0xfffb9ef4, 0xfffac6e7, + 0x9ed866dc, 0xf7cca339, 0x91ab9fc5, 0x48eead19, + 0xcb3defe5, 0x42fd12bf, 0xc9da1ef4, 0x8157a6e7, + 0x0025cbf7, 0x0025cbf7, 0x00064593, 0x00064593, + 0x00385bb5, 0x00385bb5, 0x002d485d, 0x002d485d, + 0x9a24abf7, 0x9a24abf7, 0x4b38a593, 0x4b38a593, + 0x052efbb5, 0x052efbb5, 0xeeb8e85d, 0xeeb8e85d, + 0xffd69161, 0xffd69161, 0xffdf39c8, 0xffdf39c8, + 0x000f017b, 0x000f017b, 0xffcbef0e, 0xffcbef0e, + 0xa582b161, 0xa582b161, 0x231839c8, 0x231839c8, + 0x0ebe617b, 0x0ebe617b, 0x2eadaf0e, 0x2eadaf0e, + 0x002bc1bf, 0x002bc1bf, 0x002bc1bf, 0x002bc1bf, + 0xffc9756a, 0xffc9756a, 0xffc9756a, 0xffc9756a, + 0x94e3a1bf, 0x94e3a1bf, 0x94e3a1bf, 0x94e3a1bf, + 0x2176b56a, 0x2176b56a, 0x2176b56a, 0x2176b56a, + 0x002e7184, 0x002e7184, 0x002e7184, 0x002e7184, + 0x003aea7b, 0x003aea7b, 0x003aea7b, 0x003aea7b, + 0x1c5ef184, 0x1c5ef184, 0x1c5ef184, 0x1c5ef184, + 0x0c0a4a7b, 0x0c0a4a7b, 0x0c0a4a7b, 0x0c0a4a7b, + 0x00111560, 0x00111560, 0x00111560, 0x00111560, + 0x00111560, 0x00111560, 0x00111560, 0x00111560, + 0xf2bd1560, 0xf2bd1560, 0xf2bd1560, 0xf2bd1560, + 0xf2bd1560, 0xf2bd1560, 0xf2bd1560, 0xf2bd1560, + 0x00086270, 0x00086270, 0x00086270, 0x00086270, + 0x00086270, 0x00086270, 0x00086270, 0x00086270, + 0x94566270, 0x94566270, 0x94566270, 0x94566270, + 0x94566270, 0x94566270, 0x94566270, 0x94566270, + 0x00057b53, 0x00057b53, 0x00057b53, 0x00057b53, + 0x00057b53, 0x00057b53, 0x00057b53, 0x00057b53, + 0x51efdb53, 0x51efdb53, 0x51efdb53, 0x51efdb53, + 0x51efdb53, 0x51efdb53, 0x51efdb53, 0x51efdb53, + 0x000bdee8, 0x000bdee8, 0x000bdee8, 0x000bdee8, + 0x000bdee8, 0x000bdee8, 0x000bdee8, 0x000bdee8, + 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, + 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, + 0x0036de3e, 0x000bba6e, 0x0008032a, 0x00364683, + 0xffcf107a, 0xffe0ff7c, 0x002fa50a, 0x0009ffdf, + 0xb4fe9e3e, 0xf8597a6e, 0x136d432a, 0x9386a683, + 0x8cde507a, 0x51d07f7c, 0x97d0e50a, 0xcc85dfdf, + 0x0007f904, 0x0000a8fc, 0x00189d76, 0xfff8707d, + 0xfff380a6, 0xfff21f1a, 0xffe3a1e6, 0xfff241a2, + 0x8d287904, 0x872028fc, 0x30c75d76, 0xc388107d, + 0xb50840a6, 0x1ed55f1a, 0x192061e6, 0xff2681a2, + 0xfffe1036, 0xfffe1036, 0x00376f20, 0x00376f20, + 0x00302d52, 0x00302d52, 0x0030ad80, 0x0030ad80, + 0x7f04d036, 0x7f04d036, 0xde1b6f20, 0xde1b6f20, + 0xa4da6d52, 0xa4da6d52, 0x55e0ad80, 0x55e0ad80, + 0x000f430a, 0x000f430a, 0x003e4f8e, 0x003e4f8e, + 0xffe2688e, 0xffe2688e, 0x0013308b, 0x0013308b, + 0x8b70830a, 0x8b70830a, 0x3b300f8e, 0x3b300f8e, + 0x3df4288e, 0x3df4288e, 0x4ca4908b, 0x4ca4908b, + 0xffc44151, 0xffc44151, 0xffc44151, 0xffc44151, + 0x0026b82c, 0x0026b82c, 0x0026b82c, 0x0026b82c, + 0x236e6151, 0x236e6151, 0x236e6151, 0x236e6151, + 0x712c382c, 0x712c382c, 0x712c382c, 0x712c382c, + 0x0036cfd4, 0x0036cfd4, 0x0036cfd4, 0x0036cfd4, + 0x00195afd, 0x00195afd, 0x00195afd, 0x00195afd, + 0x40314fd4, 0x40314fd4, 0x40314fd4, 0x40314fd4, + 0xa0f8fafd, 0xa0f8fafd, 0xa0f8fafd, 0xa0f8fafd, + 0xffc94878, 0xffc94878, 0xffc94878, 0xffc94878, + 0xffc94878, 0xffc94878, 0xffc94878, 0xffc94878, + 0xccd84878, 0xccd84878, 0xccd84878, 0xccd84878, + 0xccd84878, 0xccd84878, 0xccd84878, 0xccd84878, + 0x00107a5c, 0x00107a5c, 0x00107a5c, 0x00107a5c, + 0x00107a5c, 0x00107a5c, 0x00107a5c, 0x00107a5c, + 0x515bfa5c, 0x515bfa5c, 0x515bfa5c, 0x515bfa5c, + 0x515bfa5c, 0x515bfa5c, 0x515bfa5c, 0x515bfa5c, + 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, + 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, + 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, + 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, + 0x0030ba22, 0x001244aa, 0x00395d04, 0x0035b760, + 0xffca64a3, 0x0012db10, 0xffdada79, 0xfffbed0b, + 0x8e74fa22, 0x9ba784aa, 0xb9d9dd04, 0x8721b760, + 0x86dec4a3, 0x1374db10, 0x02a9fa79, 0x241d4d0b, + 0x00365bde, 0x00255461, 0xffddc205, 0x0033008e, + 0xffc5be08, 0xffdca72c, 0xffcc00a6, 0xffe0156d, + 0x54b21bde, 0xfe317461, 0xc99e6205, 0x5144c08e, + 0xd386be08, 0xaec2272c, 0xc4e0c0a6, 0x000db56d, + 0x00183045, 0x00183045, 0xffdeca39, 0xffdeca39, + 0xffcaf612, 0xffcaf612, 0x001629a3, 0x001629a3, + 0xf7a0d045, 0xf7a0d045, 0xa0a5ea39, 0xa0a5ea39, + 0x9d8d3612, 0x9d8d3612, 0x7fca89a3, 0x7fca89a3, + 0x002e67e7, 0x002e67e7, 0x00381e31, 0x00381e31, + 0x0017537f, 0x0017537f, 0x003bf91b, 0x003bf91b, + 0x75ab47e7, 0x75ab47e7, 0xaf7e3e31, 0xaf7e3e31, + 0x2707337f, 0x2707337f, 0x5ddf591b, 0x5ddf591b, + 0xffca213b, 0xffca213b, 0xffca213b, 0xffca213b, + 0xffd10b33, 0xffd10b33, 0xffd10b33, 0xffd10b33, + 0x9271813b, 0x9271813b, 0x9271813b, 0x9271813b, + 0x53b76b33, 0x53b76b33, 0x53b76b33, 0x53b76b33, + 0xfffe89e0, 0xfffe89e0, 0xfffe89e0, 0xfffe89e0, + 0xffd6b599, 0xffd6b599, 0xffd6b599, 0xffd6b599, + 0x613a89e0, 0x613a89e0, 0x613a89e0, 0x613a89e0, + 0x6e09d599, 0x6e09d599, 0x6e09d599, 0x6e09d599, + 0xfff05f90, 0xfff05f90, 0xfff05f90, 0xfff05f90, + 0xfff05f90, 0xfff05f90, 0xfff05f90, 0xfff05f90, + 0x83e25f90, 0x83e25f90, 0x83e25f90, 0x83e25f90, + 0x83e25f90, 0x83e25f90, 0x83e25f90, 0x83e25f90, + 0xffd669a8, 0xffd669a8, 0xffd669a8, 0xffd669a8, + 0xffd669a8, 0xffd669a8, 0xffd669a8, 0xffd669a8, + 0x990b69a8, 0x990b69a8, 0x990b69a8, 0x990b69a8, + 0x990b69a8, 0x990b69a8, 0x990b69a8, 0x990b69a8, + 0xffe421d5, 0xffe421d5, 0xffe421d5, 0xffe421d5, + 0xffe421d5, 0xffe421d5, 0xffe421d5, 0xffe421d5, + 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, + 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, + 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, + 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, + 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, + 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, + 0x0007eafd, 0x0007eafd, 0x0007eafd, 0x0007eafd, + 0x0007eafd, 0x0007eafd, 0x0007eafd, 0x0007eafd, + 0x72e78afd, 0x72e78afd, 0x72e78afd, 0x72e78afd, + 0x72e78afd, 0x72e78afd, 0x72e78afd, 0x72e78afd, + 0x0027cefe, 0x0027cefe, 0x0027cefe, 0x0027cefe, + 0x0027cefe, 0x0027cefe, 0x0027cefe, 0x0027cefe, + 0x73078efe, 0x73078efe, 0x73078efe, 0x73078efe, + 0x73078efe, 0x73078efe, 0x73078efe, 0x73078efe, + 0xffff9b09, 0xffff9b09, 0xffff9b09, 0xffff9b09, + 0xffff9b09, 0xffff9b09, 0xffff9b09, 0xffff9b09, + 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, + 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, + 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, + 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, + 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, + 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_poly_red_avx2(sword32* a) +{ + word64 rdi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11; + + rdi = (word64)(size_t)a; + + y10 = _mm256_setzero_si256(); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_v, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_add_epi32(y0, y11); + y9 = _mm256_add_epi32(y1, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y0 = _mm256_sub_epi32(y0, y8); + y1 = _mm256_sub_epi32(y1, y9); + y8 = _mm256_add_epi32(y2, y11); + y9 = _mm256_add_epi32(y3, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y2 = _mm256_sub_epi32(y2, y8); + y3 = _mm256_sub_epi32(y3, y9); + y8 = _mm256_add_epi32(y4, y11); + y9 = _mm256_add_epi32(y5, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y4 = _mm256_sub_epi32(y4, y8); + y5 = _mm256_sub_epi32(y5, y9); + y8 = _mm256_add_epi32(y6, y11); + y9 = _mm256_add_epi32(y7, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y6 = _mm256_sub_epi32(y6, y8); + y7 = _mm256_sub_epi32(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y8 = _mm256_add_epi32(y0, y11); + y9 = _mm256_add_epi32(y1, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y0 = _mm256_sub_epi32(y0, y8); + y1 = _mm256_sub_epi32(y1, y9); + y8 = _mm256_add_epi32(y2, y11); + y9 = _mm256_add_epi32(y3, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y2 = _mm256_sub_epi32(y2, y8); + y3 = _mm256_sub_epi32(y3, y9); + y8 = _mm256_add_epi32(y4, y11); + y9 = _mm256_add_epi32(y5, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y4 = _mm256_sub_epi32(y4, y8); + y5 = _mm256_sub_epi32(y5, y9); + y8 = _mm256_add_epi32(y6, y11); + y9 = _mm256_add_epi32(y7, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y6 = _mm256_sub_epi32(y6, y8); + y7 = _mm256_sub_epi32(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y8 = _mm256_add_epi32(y0, y11); + y9 = _mm256_add_epi32(y1, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y0 = _mm256_sub_epi32(y0, y8); + y1 = _mm256_sub_epi32(y1, y9); + y8 = _mm256_add_epi32(y2, y11); + y9 = _mm256_add_epi32(y3, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y2 = _mm256_sub_epi32(y2, y8); + y3 = _mm256_sub_epi32(y3, y9); + y8 = _mm256_add_epi32(y4, y11); + y9 = _mm256_add_epi32(y5, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y4 = _mm256_sub_epi32(y4, y8); + y5 = _mm256_sub_epi32(y5, y9); + y8 = _mm256_add_epi32(y6, y11); + y9 = _mm256_add_epi32(y7, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y6 = _mm256_sub_epi32(y6, y8); + y7 = _mm256_sub_epi32(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_add_epi32(y0, y11); + y9 = _mm256_add_epi32(y1, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y0 = _mm256_sub_epi32(y0, y8); + y1 = _mm256_sub_epi32(y1, y9); + y8 = _mm256_add_epi32(y2, y11); + y9 = _mm256_add_epi32(y3, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y2 = _mm256_sub_epi32(y2, y8); + y3 = _mm256_sub_epi32(y3, y9); + y8 = _mm256_add_epi32(y4, y11); + y9 = _mm256_add_epi32(y5, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y4 = _mm256_sub_epi32(y4, y8); + y5 = _mm256_sub_epi32(y5, y9); + y8 = _mm256_add_epi32(y6, y11); + y9 = _mm256_add_epi32(y7, y11); + y8 = _mm256_srai_epi32(y8, 23); + y9 = _mm256_srai_epi32(y9, 23); + y8 = _mm256_mullo_epi32(y8, y10); + y9 = _mm256_mullo_epi32(y9, y10); + y6 = _mm256_sub_epi32(y6, y8); + y7 = _mm256_sub_epi32(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_ntt_avx2(sword32* r) +{ + word64 rdi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)r; + + y14 = _mm256_setzero_si256(); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + /* ntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx2_zetas); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + /* 128: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); + /* 128: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y7); + /* 128: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y7); + /* 128: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y7); + y4 = y1; + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + /* 32: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 1/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 1/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1024)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1056)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1088)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1120)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1184)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1216)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1248)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1280)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1312)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1344)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1376)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + /* 32: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1408)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1440)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1568)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1600)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1632)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1664)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1696)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1728)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1760)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1792)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1824)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 2/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1856)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1888)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1920)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1952)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1984)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2016)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2048)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2080)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 2/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2112)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2144)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2176)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2208)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2240)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2272)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2304)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2336)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2400)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2432)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2464)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2496)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2528)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2560)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2592)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + /* 32: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2624)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2656)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2688)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2720)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2752)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2784)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2816)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2848)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2880)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2912)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2944)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2976)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3008)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3040)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 3/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3072)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3104)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3136)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3168)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3200)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3232)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3264)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3296)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 3/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3328)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3360)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3392)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3424)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3456)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3488)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3520)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3552)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3584)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3616)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3648)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3680)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3712)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3744)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3776)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3808)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + /* 32: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3840)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3872)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3904)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3936)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3968)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4000)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4032)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4064)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4096)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4128)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4160)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4224)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4256)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 4/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4288)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4320)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4352)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4384)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4416)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4480)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4512)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 4/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4544)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4576)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4608)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4640)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4672)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4704)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4736)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4768)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4800)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4832)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_ntt_small_avx2(sword32* r) +{ + word64 rdi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)r; + + y14 = _mm256_setzero_si256(); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + /* ntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx2_zetas); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + /* 128: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_zeta1, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_half, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_mullo_epi32(y4, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); + /* 128: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_zeta1, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_half, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y8 = _mm256_mullo_epi32(y4, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y7); + /* 128: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_zeta1, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_half, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y8 = _mm256_mullo_epi32(y4, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y7); + /* 128: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_zeta1, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_half, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y8 = _mm256_mullo_epi32(y4, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y7); + y4 = y1; + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + /* 32: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 1/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 1/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1024)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1056)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1088)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1120)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1184)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1216)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1248)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1280)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1312)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1344)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1376)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + /* 32: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1408)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1440)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1568)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1600)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1632)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1664)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1696)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1728)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1760)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1792)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1824)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 2/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1856)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1888)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1920)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1952)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1984)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2016)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2048)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2080)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 2/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2112)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2144)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2176)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2208)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2240)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2272)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2304)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2336)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2400)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2432)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2464)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2496)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2528)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2560)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2592)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + /* 32: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2624)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2656)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2688)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2720)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2752)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2784)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2816)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2848)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2880)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2912)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2944)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2976)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3008)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3040)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 3/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3072)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3104)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3136)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3168)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3200)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3232)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3264)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3296)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 3/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3328)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3360)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3392)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3424)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3456)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3488)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3520)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3552)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3584)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3616)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3648)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3680)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3712)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3744)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3776)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3808)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + /* 32: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3840)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3872)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3904)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3936)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3968)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4000)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4032)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4064)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4096)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4128)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4160)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4224)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4256)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 4/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4288)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4320)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4352)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4384)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4416)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4480)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4512)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 4/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4544)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4576)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4608)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4640)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4672)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4704)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4736)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4768)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4800)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4832)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_ntt_full_avx2(sword32* r) +{ + word64 rdi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)r; + + y14 = _mm256_setzero_si256(); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + /* ntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx2_zetas); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + /* 128: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); + /* 128: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y7); + /* 128: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y7); + /* 128: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y7); + y4 = y1; + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + /* 32: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 1/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 1/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1024)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1056)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1088)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1120)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1184)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1216)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1248)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1280)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1312)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1344)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1376)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + /* 32: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1408)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1440)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1568)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1600)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1632)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1664)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1696)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1728)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1760)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1792)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1824)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 2/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1856)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1888)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1920)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1952)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1984)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2016)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2048)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2080)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 2/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2112)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2144)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2176)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2208)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2240)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2272)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2304)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2336)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2400)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2432)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2464)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2496)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2528)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2560)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2592)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + /* 32: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2624)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2656)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2688)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2720)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2752)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2784)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2816)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2848)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2880)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2912)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2944)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2976)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3008)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3040)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 3/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3072)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3104)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3136)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3168)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3200)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3232)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3264)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3296)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 3/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3328)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3360)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3392)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3424)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3456)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3488)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3520)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3552)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3584)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3616)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3648)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3680)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3712)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3744)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3776)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3808)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + /* 32: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3840)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3872)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3904)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3936)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3968)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4000)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4032)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4064)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4096)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4128)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4160)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4224)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4256)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 4/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4288)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4320)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4352)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4384)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4416)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4480)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4512)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 4/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4544)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4576)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4608)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4640)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4672)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4704)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4736)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4768)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4800)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4832)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_ntt_small_full_avx2(sword32* r) +{ + word64 rdi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)r; + + y14 = _mm256_setzero_si256(); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + /* ntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx2_zetas); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + /* 128: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_zeta1, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_half, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_mullo_epi32(y4, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); + /* 128: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_zeta1, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_half, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y8 = _mm256_mullo_epi32(y4, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y7); + /* 128: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_zeta1, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_half, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y8 = _mm256_mullo_epi32(y4, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y7); + /* 128: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_zeta1, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_avx2_half, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y8 = _mm256_mullo_epi32(y4, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y10); + y9 = _mm256_add_epi32(y8, y12); + y9 = _mm256_srai_epi32(y9, 23); + y9 = _mm256_mullo_epi32(y9, y14); + y8 = _mm256_sub_epi32(y8, y9); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 64: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_mullo_epi32(y2, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y7); + y4 = y1; + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + /* 32: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 1/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 1/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1024)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1056)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1088)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1120)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1184)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1216)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1248)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1280)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1312)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1344)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1376)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + /* 32: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1408)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1440)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1568)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1600)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1632)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1664)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1696)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1728)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1760)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1792)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1824)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 2/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1856)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1888)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1920)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1952)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1984)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2016)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2048)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2080)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 2/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2112)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2144)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2176)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2208)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2240)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2272)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2304)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2336)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2400)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2432)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2464)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2496)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2528)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2560)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2592)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + /* 32: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2624)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2656)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2688)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2720)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2752)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2784)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2816)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2848)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2880)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2912)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2944)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2976)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3008)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3040)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 3/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3072)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3104)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3136)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3168)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3200)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3232)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3264)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3296)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 3/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3328)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3360)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3392)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3424)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3456)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3488)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3520)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3552)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3584)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3616)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3648)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3680)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3712)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3744)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3776)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3808)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + /* 32: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3840)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3872)); + y8 = _mm256_mullo_epi32(y4, y12); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y4 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y12); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y8 = _mm256_mullo_epi32(y7, y12); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y3, y8); + y3 = _mm256_add_epi32(y3, y8); + /* 16: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3904)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3936)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3968)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4000)); + y8 = _mm256_mullo_epi32(y2, y12); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y2 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y12); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y1, y8); + y1 = _mm256_add_epi32(y1, y8); + y8 = _mm256_mullo_epi32(y6, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y6 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y5, y8); + y5 = _mm256_add_epi32(y5, y8); + /* 8: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4032)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4064)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4096)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4128)); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4160)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4224)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4256)); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 4: 4/4 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4288)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4320)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4352)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4384)); + y0 = _mm256_mullo_epi32(y1, y12); + y2 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y1, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y1 = _mm256_sub_epi32(y8, y0); + y8 = _mm256_add_epi32(y8, y0); + y0 = _mm256_mullo_epi32(y3, y13); + y2 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y14); + y15 = _mm256_mul_epi32(y15, y14); + y0 = _mm256_sub_epi64(y3, y0); + y15 = _mm256_sub_epi64(y2, y15); + y0 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_blend_epi32(y0, y15, 0xaa); + y3 = _mm256_sub_epi32(y9, y0); + y9 = _mm256_add_epi32(y9, y0); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4416)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4480)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4512)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 4: 4/4 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4544)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4576)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4608)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4640)); + y4 = _mm256_mullo_epi32(y5, y12); + y6 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y5, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y5 = _mm256_sub_epi32(y8, y4); + y8 = _mm256_add_epi32(y8, y4); + y4 = _mm256_mullo_epi32(y7, y13); + y6 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y7, y4); + y15 = _mm256_sub_epi64(y6, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y7 = _mm256_sub_epi32(y9, y4); + y9 = _mm256_add_epi32(y9, y4); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4672)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4704)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4736)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4768)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4800)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4832)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi32(y1, y12); + y9 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y1, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_sub_epi32(y0, y8); + y0 = _mm256_add_epi32(y0, y8); + y8 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y3, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_sub_epi32(y2, y8); + y2 = _mm256_add_epi32(y2, y8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi32(y5, y12); + y9 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y5, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_sub_epi32(y4, y8); + y4 = _mm256_add_epi32(y4, y8); + y8 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y7, y8); + y15 = _mm256_sub_epi64(y9, y15); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_sub_epi32(y6, y8); + y6 = _mm256_add_epi32(y6, y8); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_invntt_avx2(sword32* r) +{ + word64 rdi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)r; + + y14 = _mm256_setzero_si256(); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + /* invntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx2_zetas_inv); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_shuffle_epi32(y0, 0xd8); + y9 = _mm256_shuffle_epi32(y1, 0xd8); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y2, 0xd8); + y9 = _mm256_shuffle_epi32(y3, 0xd8); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 4: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y1 = _mm256_unpackhi_epi64(y0, y1); + y9 = _mm256_unpacklo_epi64(y2, y3); + y3 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_sub_epi32(y8, y1); + y8 = _mm256_add_epi32(y8, y1); + y1 = _mm256_mullo_epi32(y0, y12); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y0, y1); + y15 = _mm256_sub_epi64(y2, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y0 = _mm256_sub_epi32(y9, y3); + y9 = _mm256_add_epi32(y9, y3); + y3 = _mm256_mullo_epi32(y0, y13); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y0, y3); + y15 = _mm256_sub_epi64(y2, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 8: 1/4 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 16: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y8 = _mm256_shuffle_epi32(y4, 0xd8); + y9 = _mm256_shuffle_epi32(y5, 0xd8); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y6, 0xd8); + y9 = _mm256_shuffle_epi32(y7, 0xd8); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 4: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y8 = _mm256_unpacklo_epi64(y4, y5); + y5 = _mm256_unpackhi_epi64(y4, y5); + y9 = _mm256_unpacklo_epi64(y6, y7); + y7 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_sub_epi32(y8, y5); + y8 = _mm256_add_epi32(y8, y5); + y5 = _mm256_mullo_epi32(y4, y12); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y4, y5); + y15 = _mm256_sub_epi64(y6, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y4 = _mm256_sub_epi32(y9, y7); + y9 = _mm256_add_epi32(y9, y7); + y7 = _mm256_mullo_epi32(y4, y13); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y4, y7); + y15 = _mm256_sub_epi64(y6, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 8: 1/4 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1024)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1056)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 16: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1088)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1120)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 32: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1184)); + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1216)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1248)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1280)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1312)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1344)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1376)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1408)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1440)); + y8 = _mm256_shuffle_epi32(y0, 0xd8); + y9 = _mm256_shuffle_epi32(y1, 0xd8); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y2, 0xd8); + y9 = _mm256_shuffle_epi32(y3, 0xd8); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 4: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1568)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y1 = _mm256_unpackhi_epi64(y0, y1); + y9 = _mm256_unpacklo_epi64(y2, y3); + y3 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_sub_epi32(y8, y1); + y8 = _mm256_add_epi32(y8, y1); + y1 = _mm256_mullo_epi32(y0, y12); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y0, y1); + y15 = _mm256_sub_epi64(y2, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y0 = _mm256_sub_epi32(y9, y3); + y9 = _mm256_add_epi32(y9, y3); + y3 = _mm256_mullo_epi32(y0, y13); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y0, y3); + y15 = _mm256_sub_epi64(y2, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 8: 2/4 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1600)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1632)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1664)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1696)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 16: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1728)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1760)); + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1792)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1824)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1856)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1888)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1920)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1952)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1984)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2016)); + y8 = _mm256_shuffle_epi32(y4, 0xd8); + y9 = _mm256_shuffle_epi32(y5, 0xd8); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y6, 0xd8); + y9 = _mm256_shuffle_epi32(y7, 0xd8); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 4: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2048)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2080)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2112)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2144)); + y8 = _mm256_unpacklo_epi64(y4, y5); + y5 = _mm256_unpackhi_epi64(y4, y5); + y9 = _mm256_unpacklo_epi64(y6, y7); + y7 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_sub_epi32(y8, y5); + y8 = _mm256_add_epi32(y8, y5); + y5 = _mm256_mullo_epi32(y4, y12); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y4, y5); + y15 = _mm256_sub_epi64(y6, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y4 = _mm256_sub_epi32(y9, y7); + y9 = _mm256_add_epi32(y9, y7); + y7 = _mm256_mullo_epi32(y4, y13); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y4, y7); + y15 = _mm256_sub_epi64(y6, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 8: 2/4 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2176)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2208)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2240)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2272)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 16: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2304)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2336)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 32: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2400)); + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2432)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2464)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2496)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2528)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2560)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2592)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2624)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2656)); + y8 = _mm256_shuffle_epi32(y0, 0xd8); + y9 = _mm256_shuffle_epi32(y1, 0xd8); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y2, 0xd8); + y9 = _mm256_shuffle_epi32(y3, 0xd8); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 4: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2688)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2720)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2752)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2784)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y1 = _mm256_unpackhi_epi64(y0, y1); + y9 = _mm256_unpacklo_epi64(y2, y3); + y3 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_sub_epi32(y8, y1); + y8 = _mm256_add_epi32(y8, y1); + y1 = _mm256_mullo_epi32(y0, y12); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y0, y1); + y15 = _mm256_sub_epi64(y2, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y0 = _mm256_sub_epi32(y9, y3); + y9 = _mm256_add_epi32(y9, y3); + y3 = _mm256_mullo_epi32(y0, y13); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y0, y3); + y15 = _mm256_sub_epi64(y2, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 8: 3/4 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2816)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2848)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2880)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2912)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 16: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2944)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2976)); + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3008)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3040)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3072)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3104)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3136)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3168)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3200)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3232)); + y8 = _mm256_shuffle_epi32(y4, 0xd8); + y9 = _mm256_shuffle_epi32(y5, 0xd8); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y6, 0xd8); + y9 = _mm256_shuffle_epi32(y7, 0xd8); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 4: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3264)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3296)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3328)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3360)); + y8 = _mm256_unpacklo_epi64(y4, y5); + y5 = _mm256_unpackhi_epi64(y4, y5); + y9 = _mm256_unpacklo_epi64(y6, y7); + y7 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_sub_epi32(y8, y5); + y8 = _mm256_add_epi32(y8, y5); + y5 = _mm256_mullo_epi32(y4, y12); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y4, y5); + y15 = _mm256_sub_epi64(y6, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y4 = _mm256_sub_epi32(y9, y7); + y9 = _mm256_add_epi32(y9, y7); + y7 = _mm256_mullo_epi32(y4, y13); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y4, y7); + y15 = _mm256_sub_epi64(y6, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 8: 3/4 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3392)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3424)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3456)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3488)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 16: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3520)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3552)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 32: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3584)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3616)); + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3648)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3680)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3712)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3744)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3776)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3808)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3840)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3872)); + y8 = _mm256_shuffle_epi32(y0, 0xd8); + y9 = _mm256_shuffle_epi32(y1, 0xd8); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y2, 0xd8); + y9 = _mm256_shuffle_epi32(y3, 0xd8); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 4: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3904)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3936)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3968)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4000)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y1 = _mm256_unpackhi_epi64(y0, y1); + y9 = _mm256_unpacklo_epi64(y2, y3); + y3 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_sub_epi32(y8, y1); + y8 = _mm256_add_epi32(y8, y1); + y1 = _mm256_mullo_epi32(y0, y12); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y0, y1); + y15 = _mm256_sub_epi64(y2, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y0 = _mm256_sub_epi32(y9, y3); + y9 = _mm256_add_epi32(y9, y3); + y3 = _mm256_mullo_epi32(y0, y13); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y0, y3); + y15 = _mm256_sub_epi64(y2, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 8: 4/4 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4032)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4064)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4096)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4128)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 16: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4160)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4192)); + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4224)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4288)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4320)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4352)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4416)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4448)); + y8 = _mm256_shuffle_epi32(y4, 0xd8); + y9 = _mm256_shuffle_epi32(y5, 0xd8); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y6, 0xd8); + y9 = _mm256_shuffle_epi32(y7, 0xd8); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 4: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4480)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4512)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4544)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4576)); + y8 = _mm256_unpacklo_epi64(y4, y5); + y5 = _mm256_unpackhi_epi64(y4, y5); + y9 = _mm256_unpacklo_epi64(y6, y7); + y7 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_sub_epi32(y8, y5); + y8 = _mm256_add_epi32(y8, y5); + y5 = _mm256_mullo_epi32(y4, y12); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y4, y5); + y15 = _mm256_sub_epi64(y6, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y4 = _mm256_sub_epi32(y9, y7); + y9 = _mm256_add_epi32(y9, y7); + y7 = _mm256_mullo_epi32(y4, y13); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y4, y7); + y15 = _mm256_sub_epi64(y6, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 8: 4/4 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4608)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4640)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4672)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4704)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 16: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4736)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4768)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 32: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4800)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4832)); + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y6); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y6 = y3; + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + /* 64: 4/4 */ + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 128: 4/4 */ + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + y8 = _mm256_mullo_epi32(y0, y13); + y10 = _mm256_mullo_epi32(y1, y13); + y9 = _mm256_shuffle_epi32(y0, 0xf5); + y12 = _mm256_shuffle_epi32(y1, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y1 = _mm256_mul_epi32(y1, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y0, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y1, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y2, y13); + y10 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y12 = _mm256_shuffle_epi32(y3, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y3, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y4, y13); + y10 = _mm256_mullo_epi32(y5, y13); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y12 = _mm256_shuffle_epi32(y5, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y5 = _mm256_mul_epi32(y5, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y5, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y4 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y6, y13); + y10 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y12 = _mm256_shuffle_epi32(y7, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y7, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y6 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_blend_epi32(y10, y9, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + /* 64: 3/4 */ + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 128: 3/4 */ + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + y8 = _mm256_mullo_epi32(y0, y13); + y10 = _mm256_mullo_epi32(y1, y13); + y9 = _mm256_shuffle_epi32(y0, 0xf5); + y12 = _mm256_shuffle_epi32(y1, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y1 = _mm256_mul_epi32(y1, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y0, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y1, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y2, y13); + y10 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y12 = _mm256_shuffle_epi32(y3, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y3, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y4, y13); + y10 = _mm256_mullo_epi32(y5, y13); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y12 = _mm256_shuffle_epi32(y5, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y5 = _mm256_mul_epi32(y5, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y5, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y4 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y6, y13); + y10 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y12 = _mm256_shuffle_epi32(y7, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y7, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y6 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_blend_epi32(y10, y9, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y7); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + /* 64: 2/4 */ + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 128: 2/4 */ + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + y8 = _mm256_mullo_epi32(y0, y13); + y10 = _mm256_mullo_epi32(y1, y13); + y9 = _mm256_shuffle_epi32(y0, 0xf5); + y12 = _mm256_shuffle_epi32(y1, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y1 = _mm256_mul_epi32(y1, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y0, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y1, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y2, y13); + y10 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y12 = _mm256_shuffle_epi32(y3, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y3, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y4, y13); + y10 = _mm256_mullo_epi32(y5, y13); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y12 = _mm256_shuffle_epi32(y5, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y5 = _mm256_mul_epi32(y5, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y5, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y4 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y6, y13); + y10 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y12 = _mm256_shuffle_epi32(y7, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y7, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y6 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_blend_epi32(y10, y9, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y7); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + /* 64: 1/4 */ + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 128: 1/4 */ + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + y8 = _mm256_mullo_epi32(y0, y13); + y10 = _mm256_mullo_epi32(y1, y13); + y9 = _mm256_shuffle_epi32(y0, 0xf5); + y12 = _mm256_shuffle_epi32(y1, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y1 = _mm256_mul_epi32(y1, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y0, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y1, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y2, y13); + y10 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y12 = _mm256_shuffle_epi32(y3, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y3, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y4, y13); + y10 = _mm256_mullo_epi32(y5, y13); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y12 = _mm256_shuffle_epi32(y5, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y5 = _mm256_mul_epi32(y5, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y5, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y4 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y6, y13); + y10 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y12 = _mm256_shuffle_epi32(y7, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y7, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y6 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_blend_epi32(y10, y9, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_invntt_full_avx2(sword32* r) +{ + word64 rdi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)r; + + y14 = _mm256_setzero_si256(); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + /* invntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx2_zetas_inv); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_shuffle_epi32(y0, 0xd8); + y9 = _mm256_shuffle_epi32(y1, 0xd8); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y2, 0xd8); + y9 = _mm256_shuffle_epi32(y3, 0xd8); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 4: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y1 = _mm256_unpackhi_epi64(y0, y1); + y9 = _mm256_unpacklo_epi64(y2, y3); + y3 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_sub_epi32(y8, y1); + y8 = _mm256_add_epi32(y8, y1); + y1 = _mm256_mullo_epi32(y0, y12); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y0, y1); + y15 = _mm256_sub_epi64(y2, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y0 = _mm256_sub_epi32(y9, y3); + y9 = _mm256_add_epi32(y9, y3); + y3 = _mm256_mullo_epi32(y0, y13); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y0, y3); + y15 = _mm256_sub_epi64(y2, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 8: 1/4 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 16: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y4 = _mm256_permute4x64_epi64(y4, 0xd8); + y5 = _mm256_permute4x64_epi64(y5, 0xd8); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y6 = _mm256_permute4x64_epi64(y6, 0xd8); + y7 = _mm256_permute4x64_epi64(y7, 0xd8); + /* 1: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 2: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y8 = _mm256_shuffle_epi32(y4, 0xd8); + y9 = _mm256_shuffle_epi32(y5, 0xd8); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y6, 0xd8); + y9 = _mm256_shuffle_epi32(y7, 0xd8); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 4: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y8 = _mm256_unpacklo_epi64(y4, y5); + y5 = _mm256_unpackhi_epi64(y4, y5); + y9 = _mm256_unpacklo_epi64(y6, y7); + y7 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_sub_epi32(y8, y5); + y8 = _mm256_add_epi32(y8, y5); + y5 = _mm256_mullo_epi32(y4, y12); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y4, y5); + y15 = _mm256_sub_epi64(y6, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y4 = _mm256_sub_epi32(y9, y7); + y9 = _mm256_add_epi32(y9, y7); + y7 = _mm256_mullo_epi32(y4, y13); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y4, y7); + y15 = _mm256_sub_epi64(y6, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 8: 1/4 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1024)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1056)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 16: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1088)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1120)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 32: 1/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1184)); + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1216)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1248)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1280)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1312)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1344)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1376)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1408)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1440)); + y8 = _mm256_shuffle_epi32(y0, 0xd8); + y9 = _mm256_shuffle_epi32(y1, 0xd8); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y2, 0xd8); + y9 = _mm256_shuffle_epi32(y3, 0xd8); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 4: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1568)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y1 = _mm256_unpackhi_epi64(y0, y1); + y9 = _mm256_unpacklo_epi64(y2, y3); + y3 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_sub_epi32(y8, y1); + y8 = _mm256_add_epi32(y8, y1); + y1 = _mm256_mullo_epi32(y0, y12); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y0, y1); + y15 = _mm256_sub_epi64(y2, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y0 = _mm256_sub_epi32(y9, y3); + y9 = _mm256_add_epi32(y9, y3); + y3 = _mm256_mullo_epi32(y0, y13); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y0, y3); + y15 = _mm256_sub_epi64(y2, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 8: 2/4 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1600)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1632)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1664)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1696)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 16: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1728)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1760)); + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y4 = _mm256_permute4x64_epi64(y4, 0xd8); + y5 = _mm256_permute4x64_epi64(y5, 0xd8); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y6 = _mm256_permute4x64_epi64(y6, 0xd8); + y7 = _mm256_permute4x64_epi64(y7, 0xd8); + /* 1: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1792)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1824)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1856)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1888)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 2: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1920)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1952)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1984)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2016)); + y8 = _mm256_shuffle_epi32(y4, 0xd8); + y9 = _mm256_shuffle_epi32(y5, 0xd8); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y6, 0xd8); + y9 = _mm256_shuffle_epi32(y7, 0xd8); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 4: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2048)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2080)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2112)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2144)); + y8 = _mm256_unpacklo_epi64(y4, y5); + y5 = _mm256_unpackhi_epi64(y4, y5); + y9 = _mm256_unpacklo_epi64(y6, y7); + y7 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_sub_epi32(y8, y5); + y8 = _mm256_add_epi32(y8, y5); + y5 = _mm256_mullo_epi32(y4, y12); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y4, y5); + y15 = _mm256_sub_epi64(y6, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y4 = _mm256_sub_epi32(y9, y7); + y9 = _mm256_add_epi32(y9, y7); + y7 = _mm256_mullo_epi32(y4, y13); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y4, y7); + y15 = _mm256_sub_epi64(y6, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 8: 2/4 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2176)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2208)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2240)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2272)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 16: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2304)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2336)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 32: 2/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2400)); + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2432)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2464)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2496)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2528)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2560)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2592)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2624)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2656)); + y8 = _mm256_shuffle_epi32(y0, 0xd8); + y9 = _mm256_shuffle_epi32(y1, 0xd8); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y2, 0xd8); + y9 = _mm256_shuffle_epi32(y3, 0xd8); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 4: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2688)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2720)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2752)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2784)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y1 = _mm256_unpackhi_epi64(y0, y1); + y9 = _mm256_unpacklo_epi64(y2, y3); + y3 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_sub_epi32(y8, y1); + y8 = _mm256_add_epi32(y8, y1); + y1 = _mm256_mullo_epi32(y0, y12); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y0, y1); + y15 = _mm256_sub_epi64(y2, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y0 = _mm256_sub_epi32(y9, y3); + y9 = _mm256_add_epi32(y9, y3); + y3 = _mm256_mullo_epi32(y0, y13); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y0, y3); + y15 = _mm256_sub_epi64(y2, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 8: 3/4 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2816)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2848)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2880)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2912)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 16: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2944)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2976)); + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y4 = _mm256_permute4x64_epi64(y4, 0xd8); + y5 = _mm256_permute4x64_epi64(y5, 0xd8); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y6 = _mm256_permute4x64_epi64(y6, 0xd8); + y7 = _mm256_permute4x64_epi64(y7, 0xd8); + /* 1: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3008)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3040)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3072)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3104)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 2: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3136)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3168)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3200)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3232)); + y8 = _mm256_shuffle_epi32(y4, 0xd8); + y9 = _mm256_shuffle_epi32(y5, 0xd8); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y6, 0xd8); + y9 = _mm256_shuffle_epi32(y7, 0xd8); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 4: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3264)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3296)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3328)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3360)); + y8 = _mm256_unpacklo_epi64(y4, y5); + y5 = _mm256_unpackhi_epi64(y4, y5); + y9 = _mm256_unpacklo_epi64(y6, y7); + y7 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_sub_epi32(y8, y5); + y8 = _mm256_add_epi32(y8, y5); + y5 = _mm256_mullo_epi32(y4, y12); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y4, y5); + y15 = _mm256_sub_epi64(y6, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y4 = _mm256_sub_epi32(y9, y7); + y9 = _mm256_add_epi32(y9, y7); + y7 = _mm256_mullo_epi32(y4, y13); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y4, y7); + y15 = _mm256_sub_epi64(y6, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 8: 3/4 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3392)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3424)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3456)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3488)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 16: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3520)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3552)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 32: 3/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3584)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3616)); + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3648)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3680)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3712)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3744)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3776)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3808)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3840)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3872)); + y8 = _mm256_shuffle_epi32(y0, 0xd8); + y9 = _mm256_shuffle_epi32(y1, 0xd8); + y0 = _mm256_unpacklo_epi32(y8, y9); + y1 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y2, 0xd8); + y9 = _mm256_shuffle_epi32(y3, 0xd8); + y2 = _mm256_unpacklo_epi32(y8, y9); + y3 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 4: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3904)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3936)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3968)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4000)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y1 = _mm256_unpackhi_epi64(y0, y1); + y9 = _mm256_unpacklo_epi64(y2, y3); + y3 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_sub_epi32(y8, y1); + y8 = _mm256_add_epi32(y8, y1); + y1 = _mm256_mullo_epi32(y0, y12); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y10); + y2 = _mm256_mul_epi32(y2, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y0, y1); + y15 = _mm256_sub_epi64(y2, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y0 = _mm256_sub_epi32(y9, y3); + y9 = _mm256_add_epi32(y9, y3); + y3 = _mm256_mullo_epi32(y0, y13); + y2 = _mm256_shuffle_epi32(y0, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y2 = _mm256_mul_epi32(y2, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y0, y3); + y15 = _mm256_sub_epi64(y2, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 8: 4/4 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4032)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4064)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4096)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4128)); + y8 = _mm256_sub_epi32(y0, y1); + y0 = _mm256_add_epi32(y0, y1); + y1 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_mul_epi32(y1, y14); + y15 = _mm256_mul_epi32(y15, y14); + y1 = _mm256_sub_epi64(y8, y1); + y15 = _mm256_sub_epi64(y9, y15); + y1 = _mm256_shuffle_epi32(y1, 0xf5); + y1 = _mm256_blend_epi32(y1, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y3); + y2 = _mm256_add_epi32(y2, y3); + y3 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + /* 16: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4160)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4192)); + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y4 = _mm256_permute4x64_epi64(y4, 0xd8); + y5 = _mm256_permute4x64_epi64(y5, 0xd8); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y6 = _mm256_permute4x64_epi64(y6, 0xd8); + y7 = _mm256_permute4x64_epi64(y7, 0xd8); + /* 1: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4224)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4288)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4320)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y11 = _mm256_shuffle_epi32(y11, 0xf5); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 2: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4352)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4416)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4448)); + y8 = _mm256_shuffle_epi32(y4, 0xd8); + y9 = _mm256_shuffle_epi32(y5, 0xd8); + y4 = _mm256_unpacklo_epi32(y8, y9); + y5 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_shuffle_epi32(y6, 0xd8); + y9 = _mm256_shuffle_epi32(y7, 0xd8); + y6 = _mm256_unpacklo_epi32(y8, y9); + y7 = _mm256_unpackhi_epi32(y8, y9); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 4: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4480)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4512)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4544)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4576)); + y8 = _mm256_unpacklo_epi64(y4, y5); + y5 = _mm256_unpackhi_epi64(y4, y5); + y9 = _mm256_unpacklo_epi64(y6, y7); + y7 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_sub_epi32(y8, y5); + y8 = _mm256_add_epi32(y8, y5); + y5 = _mm256_mullo_epi32(y4, y12); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y10); + y6 = _mm256_mul_epi32(y6, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y4, y5); + y15 = _mm256_sub_epi64(y6, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y4 = _mm256_sub_epi32(y9, y7); + y9 = _mm256_add_epi32(y9, y7); + y7 = _mm256_mullo_epi32(y4, y13); + y6 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y6 = _mm256_mul_epi32(y6, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y4, y7); + y15 = _mm256_sub_epi64(y6, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 8: 4/4 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4608)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4640)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4672)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4704)); + y8 = _mm256_sub_epi32(y4, y5); + y4 = _mm256_add_epi32(y4, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y6, y7); + y6 = _mm256_add_epi32(y6, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 16: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4736)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4768)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 32: 4/4 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4800)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4832)); + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y6); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y6 = y3; + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + /* 64: 4/4 */ + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 128: 4/4 */ + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + y8 = _mm256_mullo_epi32(y0, y13); + y10 = _mm256_mullo_epi32(y1, y13); + y9 = _mm256_shuffle_epi32(y0, 0xf5); + y12 = _mm256_shuffle_epi32(y1, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y1 = _mm256_mul_epi32(y1, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y0, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y1, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y2, y13); + y10 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y12 = _mm256_shuffle_epi32(y3, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y3, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y4, y13); + y10 = _mm256_mullo_epi32(y5, y13); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y12 = _mm256_shuffle_epi32(y5, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y5 = _mm256_mul_epi32(y5, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y5, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y4 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y6, y13); + y10 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y12 = _mm256_shuffle_epi32(y7, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y7, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y6 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_blend_epi32(y10, y9, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + /* 64: 3/4 */ + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 128: 3/4 */ + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + y8 = _mm256_mullo_epi32(y0, y13); + y10 = _mm256_mullo_epi32(y1, y13); + y9 = _mm256_shuffle_epi32(y0, 0xf5); + y12 = _mm256_shuffle_epi32(y1, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y1 = _mm256_mul_epi32(y1, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y0, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y1, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y2, y13); + y10 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y12 = _mm256_shuffle_epi32(y3, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y3, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y4, y13); + y10 = _mm256_mullo_epi32(y5, y13); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y12 = _mm256_shuffle_epi32(y5, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y5 = _mm256_mul_epi32(y5, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y5, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y4 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y6, y13); + y10 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y12 = _mm256_shuffle_epi32(y7, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y7, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y6 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_blend_epi32(y10, y9, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y7); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + /* 64: 2/4 */ + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 128: 2/4 */ + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + y8 = _mm256_mullo_epi32(y0, y13); + y10 = _mm256_mullo_epi32(y1, y13); + y9 = _mm256_shuffle_epi32(y0, 0xf5); + y12 = _mm256_shuffle_epi32(y1, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y1 = _mm256_mul_epi32(y1, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y0, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y1, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y2, y13); + y10 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y12 = _mm256_shuffle_epi32(y3, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y3, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y4, y13); + y10 = _mm256_mullo_epi32(y5, y13); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y12 = _mm256_shuffle_epi32(y5, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y5 = _mm256_mul_epi32(y5, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y5, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y4 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y6, y13); + y10 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y12 = _mm256_shuffle_epi32(y7, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y7, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y6 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_blend_epi32(y10, y9, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y7); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + /* 64: 1/4 */ + y8 = _mm256_sub_epi32(y0, y2); + y0 = _mm256_add_epi32(y0, y2); + y2 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_mul_epi32(y2, y14); + y15 = _mm256_mul_epi32(y15, y14); + y2 = _mm256_sub_epi64(y8, y2); + y15 = _mm256_sub_epi64(y9, y15); + y2 = _mm256_shuffle_epi32(y2, 0xf5); + y2 = _mm256_blend_epi32(y2, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y3); + y1 = _mm256_add_epi32(y1, y3); + y3 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_mul_epi32(y3, y14); + y15 = _mm256_mul_epi32(y15, y14); + y3 = _mm256_sub_epi64(y8, y3); + y15 = _mm256_sub_epi64(y9, y15); + y3 = _mm256_shuffle_epi32(y3, 0xf5); + y3 = _mm256_blend_epi32(y3, y15, 0xaa); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y8 = _mm256_sub_epi32(y4, y6); + y4 = _mm256_add_epi32(y4, y6); + y6 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y5, y7); + y5 = _mm256_add_epi32(y5, y7); + y7 = _mm256_mullo_epi32(y8, y13); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y11); + y9 = _mm256_mul_epi32(y9, y11); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + /* 128: 1/4 */ + y8 = _mm256_sub_epi32(y0, y4); + y0 = _mm256_add_epi32(y0, y4); + y4 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_mul_epi32(y4, y14); + y15 = _mm256_mul_epi32(y15, y14); + y4 = _mm256_sub_epi64(y8, y4); + y15 = _mm256_sub_epi64(y9, y15); + y4 = _mm256_shuffle_epi32(y4, 0xf5); + y4 = _mm256_blend_epi32(y4, y15, 0xaa); + y8 = _mm256_sub_epi32(y1, y5); + y1 = _mm256_add_epi32(y1, y5); + y5 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_mul_epi32(y5, y14); + y15 = _mm256_mul_epi32(y15, y14); + y5 = _mm256_sub_epi64(y8, y5); + y15 = _mm256_sub_epi64(y9, y15); + y5 = _mm256_shuffle_epi32(y5, 0xf5); + y5 = _mm256_blend_epi32(y5, y15, 0xaa); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y8 = _mm256_sub_epi32(y2, y6); + y2 = _mm256_add_epi32(y2, y6); + y6 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_mul_epi32(y6, y14); + y15 = _mm256_mul_epi32(y15, y14); + y6 = _mm256_sub_epi64(y8, y6); + y15 = _mm256_sub_epi64(y9, y15); + y6 = _mm256_shuffle_epi32(y6, 0xf5); + y6 = _mm256_blend_epi32(y6, y15, 0xaa); + y8 = _mm256_sub_epi32(y3, y7); + y3 = _mm256_add_epi32(y3, y7); + y7 = _mm256_mullo_epi32(y8, y12); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y10); + y9 = _mm256_mul_epi32(y9, y10); + y15 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_mul_epi32(y7, y14); + y15 = _mm256_mul_epi32(y15, y14); + y7 = _mm256_sub_epi64(y8, y7); + y15 = _mm256_sub_epi64(y9, y15); + y7 = _mm256_shuffle_epi32(y7, 0xf5); + y7 = _mm256_blend_epi32(y7, y15, 0xaa); + y8 = _mm256_mullo_epi32(y0, y13); + y10 = _mm256_mullo_epi32(y1, y13); + y9 = _mm256_shuffle_epi32(y0, 0xf5); + y12 = _mm256_shuffle_epi32(y1, 0xf5); + y0 = _mm256_mul_epi32(y0, y11); + y1 = _mm256_mul_epi32(y1, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y0, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y1, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_blend_epi32(y8, y15, 0xaa); + y1 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y2, y13); + y10 = _mm256_mullo_epi32(y3, y13); + y9 = _mm256_shuffle_epi32(y2, 0xf5); + y12 = _mm256_shuffle_epi32(y3, 0xf5); + y2 = _mm256_mul_epi32(y2, y11); + y3 = _mm256_mul_epi32(y3, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y2, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y3, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_blend_epi32(y8, y15, 0xaa); + y3 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y4, y13); + y10 = _mm256_mullo_epi32(y5, y13); + y9 = _mm256_shuffle_epi32(y4, 0xf5); + y12 = _mm256_shuffle_epi32(y5, 0xf5); + y4 = _mm256_mul_epi32(y4, y11); + y5 = _mm256_mul_epi32(y5, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y4, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y5, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y4 = _mm256_blend_epi32(y8, y15, 0xaa); + y5 = _mm256_blend_epi32(y10, y9, 0xaa); + y8 = _mm256_mullo_epi32(y6, y13); + y10 = _mm256_mullo_epi32(y7, y13); + y9 = _mm256_shuffle_epi32(y6, 0xf5); + y12 = _mm256_shuffle_epi32(y7, 0xf5); + y6 = _mm256_mul_epi32(y6, y11); + y7 = _mm256_mul_epi32(y7, y11); + y9 = _mm256_mul_epi32(y9, y11); + y12 = _mm256_mul_epi32(y12, y11); + y15 = _mm256_shuffle_epi32(y8, 0xf5); + y8 = _mm256_mul_epi32(y8, y14); + y15 = _mm256_mul_epi32(y15, y14); + y8 = _mm256_sub_epi64(y6, y8); + y15 = _mm256_sub_epi64(y9, y15); + y9 = _mm256_shuffle_epi32(y10, 0xf5); + y10 = _mm256_mul_epi32(y10, y14); + y9 = _mm256_mul_epi32(y9, y14); + y10 = _mm256_sub_epi64(y7, y10); + y9 = _mm256_sub_epi64(y12, y9); + y8 = _mm256_shuffle_epi32(y8, 0xf5); + y10 = _mm256_shuffle_epi32(y10, 0xf5); + y6 = _mm256_blend_epi32(y8, y15, 0xaa); + y7 = _mm256_blend_epi32(y10, y9, 0xaa); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_mul_avx2(sword32* r, const sword32* a, + const sword32* b) +{ + word64 rdi, rsi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + y8 = _mm256_setzero_si256(); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_qinv, 0)); + /* 0..15 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + /* 16..31 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + /* 32..47 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y2); + /* 48..63 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y2); + /* 64..79 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y2); + /* 80..95 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y2); + /* 96..111 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y2); + /* 112..127 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y2); + /* 128..143 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 512)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 544)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y2); + /* 144..159 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 576)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y2); + /* 160..175 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 640)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 672)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y2); + /* 176..191 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 704)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 736)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y2); + /* 192..207 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 768)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 800)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y2); + /* 208..223 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 832)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y2); + /* 224..239 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 896)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 928)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y2); + /* 240..255 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 960)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 992)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y0 = _mm256_mul_epi32(y0, y4); + y1 = _mm256_mul_epi32(y1, y5); + y2 = _mm256_mul_epi32(y2, y6); + y3 = _mm256_mul_epi32(y3, y7); + /* Mont Reduce 2 */ + y4 = _mm256_mullo_epi32(y0, y9); + y5 = _mm256_mullo_epi32(y1, y9); + y6 = _mm256_mullo_epi32(y2, y9); + y7 = _mm256_mullo_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y8); + y5 = _mm256_mul_epi32(y5, y8); + y6 = _mm256_mul_epi32(y6, y8); + y7 = _mm256_mul_epi32(y7, y8); + y0 = _mm256_sub_epi32(y0, y4); + y1 = _mm256_sub_epi32(y1, y5); + y2 = _mm256_sub_epi32(y2, y6); + y3 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi64(y0, 32); + y2 = _mm256_srli_epi64(y2, 32); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_or_si256(y2, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y2); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_mul_vec_4_avx2(sword32* r, const sword32* a, + const sword32* b) +{ + word64 rdi, rsi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + y12 = _mm256_setzero_si256(); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_qinv, 0)); + /* 0..7 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1024)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1024)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2048)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3072)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2048)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3072)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + /* 8..15 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1056)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1056)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2080)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3104)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2080)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3104)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + /* 16..23 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1088)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1088)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2112)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3136)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2112)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3136)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + /* 24..31 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1120)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1120)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2144)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3168)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2144)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3168)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + /* 32..39 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1152)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1152)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2176)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3200)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2176)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3200)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + /* 40..47 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1184)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1184)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2208)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3232)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2208)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3232)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y0); + /* 48..55 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1216)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1216)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2240)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3264)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2240)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3264)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y0); + /* 56..63 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1248)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1248)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2272)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3296)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2272)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3296)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y0); + /* 64..71 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1280)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1280)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2304)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3328)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2304)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3328)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + /* 72..79 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1312)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1312)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2336)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3360)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2336)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3360)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y0); + /* 80..87 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1344)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1344)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2368)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3392)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2368)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3392)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y0); + /* 88..95 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1376)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1376)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2400)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3424)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2400)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3424)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y0); + /* 96..103 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1408)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1408)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2432)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3456)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2432)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3456)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + /* 104..111 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1440)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1440)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2464)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3488)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2464)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3488)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y0); + /* 112..119 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1472)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1472)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2496)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3520)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2496)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3520)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y0); + /* 120..127 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1504)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1504)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2528)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3552)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2528)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3552)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y0); + /* 128..135 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 512)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1536)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1536)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2560)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3584)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2560)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3584)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + /* 136..143 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1568)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1568)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2592)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3616)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2592)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3616)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y0); + /* 144..151 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 576)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1600)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1600)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2624)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3648)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2624)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3648)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y0); + /* 152..159 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 608)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1632)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1632)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2656)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3680)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2656)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3680)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y0); + /* 160..167 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 640)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1664)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1664)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2688)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3712)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2688)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3712)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y0); + /* 168..175 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 672)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1696)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1696)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2720)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3744)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2720)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3744)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y0); + /* 176..183 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 704)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1728)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1728)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2752)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3776)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2752)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3776)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y0); + /* 184..191 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 736)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1760)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1760)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2784)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3808)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2784)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3808)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y0); + /* 192..199 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 768)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1792)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1792)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2816)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3840)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2816)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3840)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + /* 200..207 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1824)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1824)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2848)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3872)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2848)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3872)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y0); + /* 208..215 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 832)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1856)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1856)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2880)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3904)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2880)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3904)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y0); + /* 216..223 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 864)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1888)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1888)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2912)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3936)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2912)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3936)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y0); + /* 224..231 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 896)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1920)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1920)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2944)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3968)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2944)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3968)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y0); + /* 232..239 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 928)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1952)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1952)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2976)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4000)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2976)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4000)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y0); + /* 240..247 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 960)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1984)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1984)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3008)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4032)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3008)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4032)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y0); + /* 248..255 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 992)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2016)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2016)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3040)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4064)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3040)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4064)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y0); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_mul_vec_5_avx2(sword32* r, const sword32* a, + const sword32* b) +{ + word64 rdi, rsi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + y12 = _mm256_setzero_si256(); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_qinv, 0)); + /* 0..7 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1024)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2048)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1024)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2048)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3072)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4096)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3072)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4096)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + /* 8..15 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1056)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2080)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1056)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2080)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3104)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4128)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3104)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4128)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + /* 16..23 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1088)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2112)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1088)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2112)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3136)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4160)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3136)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4160)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + /* 24..31 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1120)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2144)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1120)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2144)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3168)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4192)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3168)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4192)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + /* 32..39 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1152)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2176)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1152)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2176)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3200)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3200)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4224)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + /* 40..47 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1184)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2208)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1184)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2208)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3232)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3232)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4256)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y0); + /* 48..55 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1216)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2240)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1216)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2240)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3264)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4288)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3264)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4288)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y0); + /* 56..63 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1248)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2272)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1248)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2272)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3296)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4320)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3296)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4320)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y0); + /* 64..71 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1280)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2304)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1280)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2304)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3328)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4352)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3328)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4352)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + /* 72..79 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1312)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2336)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1312)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2336)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3360)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3360)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4384)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y0); + /* 80..87 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1344)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2368)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1344)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2368)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3392)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4416)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3392)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4416)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y0); + /* 88..95 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1376)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2400)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1376)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2400)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3424)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4448)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3424)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4448)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y0); + /* 96..103 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1408)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2432)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1408)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2432)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3456)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4480)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3456)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4480)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + /* 104..111 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1440)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2464)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1440)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2464)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3488)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4512)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3488)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4512)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y0); + /* 112..119 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1472)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2496)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1472)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2496)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3520)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4544)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3520)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4544)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y0); + /* 120..127 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1504)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2528)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1504)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2528)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3552)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4576)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3552)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4576)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y0); + /* 128..135 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 512)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1536)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2560)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1536)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2560)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3584)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4608)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3584)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4608)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + /* 136..143 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1568)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2592)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1568)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2592)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3616)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4640)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3616)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4640)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y0); + /* 144..151 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 576)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1600)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2624)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1600)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2624)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3648)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4672)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3648)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4672)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y0); + /* 152..159 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 608)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1632)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2656)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1632)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2656)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3680)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4704)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3680)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4704)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y0); + /* 160..167 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 640)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1664)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2688)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1664)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2688)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3712)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4736)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3712)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4736)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y0); + /* 168..175 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 672)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1696)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2720)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1696)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2720)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3744)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4768)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3744)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4768)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y0); + /* 176..183 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 704)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1728)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2752)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1728)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2752)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3776)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4800)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3776)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4800)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y0); + /* 184..191 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 736)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1760)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2784)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1760)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2784)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3808)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4832)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3808)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4832)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y0); + /* 192..199 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 768)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1792)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2816)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1792)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2816)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3840)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4864)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3840)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + /* 200..207 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1824)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2848)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1824)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2848)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3872)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4896)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3872)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y0); + /* 208..215 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 832)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1856)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2880)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1856)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2880)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3904)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4928)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3904)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y0); + /* 216..223 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 864)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1888)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2912)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1888)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2912)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3936)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4960)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3936)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y0); + /* 224..231 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 896)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1920)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2944)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1920)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2944)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3968)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4992)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3968)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y0); + /* 232..239 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 928)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1952)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2976)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1952)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2976)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4000)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5024)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4000)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y0); + /* 240..247 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 960)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1984)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3008)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1984)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3008)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4032)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5056)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4032)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y0); + /* 248..255 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 992)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2016)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3040)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2016)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3040)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4064)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5088)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4064)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y0); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_mul_vec_7_avx2(sword32* r, const sword32* a, + const sword32* b) +{ + word64 rdi, rsi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + y12 = _mm256_setzero_si256(); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_qinv, 0)); + /* 0..7 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1024)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2048)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1024)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2048)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3072)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4096)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3072)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4096)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5120)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6144)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5120)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6144)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + /* 8..15 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1056)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2080)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1056)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2080)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3104)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4128)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3104)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4128)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5152)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6176)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5152)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6176)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + /* 16..23 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1088)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2112)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1088)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2112)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3136)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4160)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3136)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4160)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5184)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6208)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5184)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6208)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + /* 24..31 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1120)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2144)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1120)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2144)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3168)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4192)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3168)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4192)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5216)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6240)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5216)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6240)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + /* 32..39 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1152)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2176)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1152)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2176)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3200)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3200)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4224)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5248)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6272)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5248)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6272)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + /* 40..47 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1184)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2208)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1184)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2208)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3232)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3232)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4256)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5280)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6304)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5280)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6304)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y0); + /* 48..55 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1216)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2240)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1216)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2240)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3264)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4288)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3264)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4288)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5312)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6336)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5312)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6336)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y0); + /* 56..63 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1248)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2272)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1248)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2272)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3296)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4320)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3296)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4320)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5344)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6368)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5344)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6368)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y0); + /* 64..71 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1280)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2304)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1280)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2304)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3328)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4352)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3328)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4352)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5376)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6400)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5376)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6400)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + /* 72..79 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1312)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2336)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1312)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2336)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3360)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3360)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4384)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5408)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6432)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5408)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6432)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y0); + /* 80..87 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1344)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2368)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1344)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2368)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3392)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4416)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3392)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4416)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5440)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6464)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5440)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6464)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y0); + /* 88..95 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1376)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2400)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1376)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2400)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3424)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4448)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3424)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4448)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5472)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6496)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5472)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6496)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y0); + /* 96..103 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1408)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2432)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1408)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2432)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3456)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4480)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3456)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4480)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5504)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6528)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5504)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6528)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + /* 104..111 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1440)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2464)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1440)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2464)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3488)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4512)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3488)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4512)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5536)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6560)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5536)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6560)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y0); + /* 112..119 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1472)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2496)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1472)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2496)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3520)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4544)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3520)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4544)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5568)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6592)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5568)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6592)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y0); + /* 120..127 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1504)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2528)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1504)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2528)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3552)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4576)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3552)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4576)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5600)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6624)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5600)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6624)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y0); + /* 128..135 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 512)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1536)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2560)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 512)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1536)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2560)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3584)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4608)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3584)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4608)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5632)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6656)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5632)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6656)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + /* 136..143 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1568)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2592)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 544)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1568)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2592)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3616)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4640)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3616)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4640)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5664)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6688)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5664)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6688)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y0); + /* 144..151 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 576)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1600)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2624)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 576)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1600)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2624)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3648)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4672)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3648)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4672)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5696)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6720)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5696)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6720)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y0); + /* 152..159 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 608)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1632)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2656)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 608)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1632)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2656)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3680)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4704)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3680)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4704)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5728)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6752)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5728)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6752)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y0); + /* 160..167 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 640)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1664)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2688)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 640)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1664)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2688)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3712)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4736)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3712)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4736)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5760)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6784)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5760)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6784)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y0); + /* 168..175 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 672)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1696)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2720)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 672)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1696)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2720)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3744)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4768)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3744)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4768)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5792)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6816)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5792)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6816)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y0); + /* 176..183 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 704)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1728)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2752)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 704)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1728)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2752)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3776)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4800)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3776)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4800)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5824)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6848)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5824)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6848)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y0); + /* 184..191 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 736)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1760)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2784)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 736)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1760)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2784)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3808)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4832)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3808)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4832)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5856)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6880)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5856)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6880)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y0); + /* 192..199 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 768)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1792)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2816)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 768)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1792)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2816)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3840)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4864)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3840)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4864)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5888)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6912)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5888)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6912)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + /* 200..207 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1824)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2848)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 800)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1824)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2848)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3872)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4896)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3872)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4896)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5920)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6944)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5920)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6944)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y0); + /* 208..215 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 832)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1856)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2880)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 832)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1856)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2880)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3904)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4928)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3904)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4928)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5952)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6976)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5952)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6976)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y0); + /* 216..223 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 864)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1888)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2912)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 864)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1888)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2912)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3936)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4960)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3936)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4960)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5984)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 7008)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5984)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 7008)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y0); + /* 224..231 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 896)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1920)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2944)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 896)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1920)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2944)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3968)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4992)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3968)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4992)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6016)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 7040)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6016)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 7040)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y0); + /* 232..239 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 928)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1952)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2976)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 928)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1952)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2976)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4000)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5024)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4000)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5024)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6048)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 7072)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6048)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 7072)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y0); + /* 240..247 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 960)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 1984)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3008)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 960)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 1984)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3008)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4032)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5056)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4032)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5056)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6080)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 7104)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6080)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 7104)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y0); + /* 248..255 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 992)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 2016)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 3040)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 992)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 2016)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 3040)); + y1 = _mm256_shuffle_epi32(y0, 0xf5); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y7 = _mm256_shuffle_epi32(y6, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y0 = _mm256_mul_epi32(y0, y6); + y1 = _mm256_mul_epi32(y1, y7); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 4064)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 5088)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 4064)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 5088)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 6112)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 7136)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 6112)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 7136)); + y3 = _mm256_shuffle_epi32(y2, 0xf5); + y5 = _mm256_shuffle_epi32(y4, 0xf5); + y9 = _mm256_shuffle_epi32(y8, 0xf5); + y11 = _mm256_shuffle_epi32(y10, 0xf5); + y2 = _mm256_mul_epi32(y2, y8); + y3 = _mm256_mul_epi32(y3, y9); + y4 = _mm256_mul_epi32(y4, y10); + y5 = _mm256_mul_epi32(y5, y11); + y0 = _mm256_add_epi64(y0, y2); + y1 = _mm256_add_epi64(y1, y3); + y0 = _mm256_add_epi64(y0, y4); + y1 = _mm256_add_epi64(y1, y5); + /* Mont Reduce 2 */ + y6 = _mm256_mullo_epi32(y0, y13); + y7 = _mm256_mullo_epi32(y1, y13); + y6 = _mm256_mul_epi32(y6, y12); + y7 = _mm256_mul_epi32(y7, y12); + y0 = _mm256_sub_epi32(y0, y6); + y1 = _mm256_sub_epi32(y1, y7); + y0 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_or_si256(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y0); +} + +XALIGNED(32) static const word64 L_mldsa_rej_idx[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000002ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000003ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000000000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000000000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000000000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000004ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000001ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000000000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000002ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000000000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000000000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000002ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000003ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000000000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000000000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000000000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000400000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000400000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000000000004ULL, 0x0000000000000000ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000001ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000002ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000002ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000003ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000500000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000500000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000500000004ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000000ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000001ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000002ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000500000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000500000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000002ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000400000003ULL, 0x0000000000000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000500000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000500000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000003ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000500000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000400000003ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000400000003ULL, + 0x0000000000000005ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000500000004ULL, 0x0000000000000000ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000600000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000600000001ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000600000002ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000600000002ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000600000003ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000600000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000600000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000600000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000600000004ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000000ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000001ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000600000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000002ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000600000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000600000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000002ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000400000003ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000600000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000600000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000003ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000600000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000400000003ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000400000003ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000600000004ULL, 0x0000000000000000ULL, + 0x0000000600000005ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000000ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000001ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000002ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000002ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000500000003ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000003ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000500000003ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000500000003ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000600000005ULL, 0x0000000000000000ULL, + 0x0000000500000004ULL, 0x0000000000000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000000ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000001ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000004ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000400000002ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000500000004ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000500000004ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000002ULL, + 0x0000000600000005ULL, 0x0000000000000000ULL, + 0x0000000400000003ULL, 0x0000000600000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000500000004ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000500000004ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000003ULL, + 0x0000000600000005ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000500000004ULL, + 0x0000000000000006ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000400000003ULL, + 0x0000000600000005ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000400000003ULL, + 0x0000000600000005ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000500000004ULL, 0x0000000000000006ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000700000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000700000001ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000700000002ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000700000002ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000700000003ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000700000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000700000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000700000003ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000700000004ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000000ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000001ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000700000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000002ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000700000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000700000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000002ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000400000003ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000700000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000700000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000003ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000700000004ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000400000003ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000400000003ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000700000004ULL, 0x0000000000000000ULL, + 0x0000000700000005ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000000ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000001ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000002ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000002ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000500000003ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000003ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000500000003ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000500000003ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000700000005ULL, 0x0000000000000000ULL, + 0x0000000500000004ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000000ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000001ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000400000002ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000500000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000500000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000002ULL, + 0x0000000700000005ULL, 0x0000000000000000ULL, + 0x0000000400000003ULL, 0x0000000700000005ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000500000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000500000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000003ULL, + 0x0000000700000005ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000500000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000400000003ULL, + 0x0000000700000005ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000400000003ULL, + 0x0000000700000005ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000500000004ULL, 0x0000000000000007ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000600000000ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000600000001ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000600000002ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000600000002ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000600000003ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000600000003ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000600000003ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000600000003ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000600000004ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000000ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000001ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000600000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000400000002ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000600000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000600000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000002ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000400000003ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000600000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000600000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000003ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000600000004ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000400000003ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000400000003ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000600000004ULL, 0x0000000000000007ULL, + 0x0000000600000005ULL, 0x0000000000000007ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000000ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000500000001ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000500000002ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000002ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000500000003ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000003ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000300000002ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000500000003ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000500000003ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000600000005ULL, 0x0000000000000007ULL, + 0x0000000500000004ULL, 0x0000000700000006ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000400000000ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000400000001ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000500000004ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000400000002ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000500000004ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000200000001ULL, 0x0000000500000004ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000002ULL, + 0x0000000600000005ULL, 0x0000000000000007ULL, + 0x0000000400000003ULL, 0x0000000600000005ULL, + 0x0000000000000007ULL, 0x0000000000000000ULL, + 0x0000000300000000ULL, 0x0000000500000004ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000300000001ULL, 0x0000000500000004ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000100000000ULL, 0x0000000400000003ULL, + 0x0000000600000005ULL, 0x0000000000000007ULL, + 0x0000000300000002ULL, 0x0000000500000004ULL, + 0x0000000700000006ULL, 0x0000000000000000ULL, + 0x0000000200000000ULL, 0x0000000400000003ULL, + 0x0000000600000005ULL, 0x0000000000000007ULL, + 0x0000000200000001ULL, 0x0000000400000003ULL, + 0x0000000600000005ULL, 0x0000000000000007ULL, + 0x0000000100000000ULL, 0x0000000300000002ULL, + 0x0000000500000004ULL, 0x0000000700000006ULL, +}; + +XALIGNED(16) static const word32 L_mldsa_rej_q[] WC_X64I_UNUSED = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +XALIGNED(32) static const word64 L_mldsa_rej_mask[] WC_X64I_UNUSED = { + 0x007fffff007fffffULL, 0x007fffff007fffffULL, + 0x007fffff007fffffULL, 0x007fffff007fffffULL, +}; + +XALIGNED(32) static const word64 L_mldsa_rej_shuffle[] WC_X64I_UNUSED = { + 0x0005040300020100ULL, 0x000b0a0900080706ULL, + 0x0009080700060504ULL, 0x000f0e0d000c0b0aULL, +}; + +XALIGNED(32) static const word64 L_mldsa_rej_ones[] WC_X64I_UNUSED = { + 0x0101010101010101ULL, 0x0101010101010101ULL, + 0x0101010101010101ULL, 0x0101010101010101ULL, +}; + +WC_X64I_TARGET("avx2,bmi,bmi2") +WOLFSSL_LOCAL int wc_mldsa_rej_uniform_n_avx2(sword32* p, word32 len, + const byte* r, word32 rLen) +{ + word64 rdi, rsi, rdx, r8, rax, r9, rbx, r10, rcx, r14 = 0, r13 = 0; + __m256i y0, y1, y2, y3, y6, y7, y8, y9; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + r8 = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_rej_q, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_rej_mask, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_rej_shuffle, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_rej_ones, 0)); + r9 = (word64)((word64)(size_t)L_mldsa_rej_idx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 24)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 48)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 72)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 120)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 144)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 168)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 216)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 240)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 264)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 312)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 336)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 360)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 408)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 432)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 456)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 504)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 528)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 552)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 576)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 600)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 624)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 648)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 672)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 696)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + rdx = (word64)(rdx + 0x2d0); + r8 = (word32)((word32)r8 - 0x2d0); +L_mldsa_rej_uniform_n_avx2_start_256: + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 24)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_and_si256(y0, y7); + y1 = _mm256_and_si256(y1, y7); + y2 = _mm256_cmpgt_epi32(y6, y0); + y3 = _mm256_cmpgt_epi32(y6, y1); + y2 = _mm256_packs_epi32(y2, y3); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y2 = _mm256_packs_epi16(y2, y2); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r10 = (word32)((word32)r10 << 5); + rcx = (word32)((word32)rcx << 5); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, rcx)); + y0 = _mm256_permutevar8x32_epi32(y0, y2); + y1 = _mm256_permutevar8x32_epi32(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rcx)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + rdi = (word64)(rdi + r10 * 4); + rsi = (word32)((word32)rsi - (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + rdi = (word64)(rdi + rcx * 4); + rsi = (word32)((word32)rsi - (word32)rcx); + rdx = (word64)(rdx + 0x30); + r8 = (word32)((word32)r8 - 0x30); + if ((sword32)((word32)r8) < (sword32)(0x30)) { + goto L_mldsa_rej_uniform_n_avx2_done_256; + } + if ((sword32)((word32)rsi) >= (sword32)(0x10)) { + goto L_mldsa_rej_uniform_n_avx2_start_256; + } +L_mldsa_rej_uniform_n_avx2_done_256: + if (((word32)rsi) == (0)) { + goto L_mldsa_rej_uniform_n_avx2_done_64; + } + r14 = (word64)(0xffffff00ffffff); + r13 = (word64)(0x7fffff007fffff); +L_mldsa_rej_uniform_n_avx2_start_64: + if ((sword32)((word32)r8) < (sword32)(8)) { + goto L_mldsa_rej_uniform_n_avx2_done_64; + } + rcx = (word64)(WC_L64(rdx, 0)); + rcx = (word64)((word64)_pdep_u64(rcx, r14)); + rcx = (word64)(rcx & r13); + if ((sword32)((word32)rcx) >= (sword32)(0x7fe001)) { + goto L_mldsa_rej_uniform_0_avx2_rej_large_0; + } + WC_S32(rdi, 0) = (word32)((word32)rcx); + rdi = (word64)(rdi + 4); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mldsa_rej_uniform_n_avx2_done_64; + } +L_mldsa_rej_uniform_0_avx2_rej_large_0: + rcx = (word64)(rcx >> 32); + if ((sword32)((word32)rcx) >= (sword32)(0x7fe001)) { + goto L_mldsa_rej_uniform_0_avx2_rej_large_1; + } + WC_S32(rdi, 0) = (word32)((word32)rcx); + rdi = (word64)(rdi + 4); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mldsa_rej_uniform_n_avx2_done_64; + } +L_mldsa_rej_uniform_0_avx2_rej_large_1: + rdx = (word64)(rdx + 6); + r8 = (word32)((word32)r8 - 6); + if ((sword32)((word32)r8) <= (sword32)(0)) { + goto L_mldsa_rej_uniform_n_avx2_done_64; + } + if ((sword32)((word32)rsi) > (sword32)(0)) { + goto L_mldsa_rej_uniform_n_avx2_start_64; + } +L_mldsa_rej_uniform_n_avx2_done_64: + rax = (word32)((word32)rax - (word32)rsi); + return (int)(word32)rax; + (void)y9; +} + +WC_X64I_TARGET("bmi2") +WOLFSSL_LOCAL int wc_mldsa_rej_uniform_avx2(sword32* p, word32 len, + const byte* r, word32 rLen) +{ + word64 rdi, rsi, rdx, r8, rax, r14 = 0, r13 = 0, rcx = 0; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + r8 = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + if (((word32)rsi) == (0)) { + goto L_mldsa_rej_uniform_avx2_done_64; + } + r14 = (word64)(0xffffff00ffffff); + r13 = (word64)(0x7fffff007fffff); +L_mldsa_rej_uniform_avx2_start_64: + if ((sword32)((word32)r8) < (sword32)(8)) { + goto L_mldsa_rej_uniform_avx2_done_64; + } + rcx = (word64)(WC_L64(rdx, 0)); + rcx = (word64)((word64)_pdep_u64(rcx, r14)); + rcx = (word64)(rcx & r13); + if ((sword32)((word32)rcx) >= (sword32)(0x7fe001)) { + goto L_mldsa_rej_uniform_avx2_rej_large_0; + } + WC_S32(rdi, 0) = (word32)((word32)rcx); + rdi = (word64)(rdi + 4); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mldsa_rej_uniform_avx2_done_64; + } +L_mldsa_rej_uniform_avx2_rej_large_0: + rcx = (word64)(rcx >> 32); + if ((sword32)((word32)rcx) >= (sword32)(0x7fe001)) { + goto L_mldsa_rej_uniform_avx2_rej_large_1; + } + WC_S32(rdi, 0) = (word32)((word32)rcx); + rdi = (word64)(rdi + 4); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mldsa_rej_uniform_avx2_done_64; + } +L_mldsa_rej_uniform_avx2_rej_large_1: + rdx = (word64)(rdx + 6); + r8 = (word32)((word32)r8 - 6); + if ((sword32)((word32)r8) <= (sword32)(0)) { + goto L_mldsa_rej_uniform_avx2_done_64; + } + if ((sword32)((word32)rsi) > (sword32)(0)) { + goto L_mldsa_rej_uniform_avx2_start_64; + } +L_mldsa_rej_uniform_avx2_done_64: + rax = (word32)((word32)rax - (word32)rsi); + return (int)(word32)rax; +} + +XALIGNED(32) static const word64 L_mldsa_shufb_rej_idx[] WC_X64I_UNUSED = { + 0xffffffffffffffffULL, 0xffffffffffffffffULL, + 0xffffffffffff0100ULL, 0xffffffffffffffffULL, + 0xffffffffffff0302ULL, 0xffffffffffffffffULL, + 0xffffffff03020100ULL, 0xffffffffffffffffULL, + 0xffffffffffff0504ULL, 0xffffffffffffffffULL, + 0xffffffff05040100ULL, 0xffffffffffffffffULL, + 0xffffffff05040302ULL, 0xffffffffffffffffULL, + 0xffff050403020100ULL, 0xffffffffffffffffULL, + 0xffffffffffff0706ULL, 0xffffffffffffffffULL, + 0xffffffff07060100ULL, 0xffffffffffffffffULL, + 0xffffffff07060302ULL, 0xffffffffffffffffULL, + 0xffff070603020100ULL, 0xffffffffffffffffULL, + 0xffffffff07060504ULL, 0xffffffffffffffffULL, + 0xffff070605040100ULL, 0xffffffffffffffffULL, + 0xffff070605040302ULL, 0xffffffffffffffffULL, + 0x0706050403020100ULL, 0xffffffffffffffffULL, + 0xffffffffffff0908ULL, 0xffffffffffffffffULL, + 0xffffffff09080100ULL, 0xffffffffffffffffULL, + 0xffffffff09080302ULL, 0xffffffffffffffffULL, + 0xffff090803020100ULL, 0xffffffffffffffffULL, + 0xffffffff09080504ULL, 0xffffffffffffffffULL, + 0xffff090805040100ULL, 0xffffffffffffffffULL, + 0xffff090805040302ULL, 0xffffffffffffffffULL, + 0x0908050403020100ULL, 0xffffffffffffffffULL, + 0xffffffff09080706ULL, 0xffffffffffffffffULL, + 0xffff090807060100ULL, 0xffffffffffffffffULL, + 0xffff090807060302ULL, 0xffffffffffffffffULL, + 0x0908070603020100ULL, 0xffffffffffffffffULL, + 0xffff090807060504ULL, 0xffffffffffffffffULL, + 0x0908070605040100ULL, 0xffffffffffffffffULL, + 0x0908070605040302ULL, 0xffffffffffffffffULL, + 0x0706050403020100ULL, 0xffffffffffff0908ULL, + 0xffffffffffff0b0aULL, 0xffffffffffffffffULL, + 0xffffffff0b0a0100ULL, 0xffffffffffffffffULL, + 0xffffffff0b0a0302ULL, 0xffffffffffffffffULL, + 0xffff0b0a03020100ULL, 0xffffffffffffffffULL, + 0xffffffff0b0a0504ULL, 0xffffffffffffffffULL, + 0xffff0b0a05040100ULL, 0xffffffffffffffffULL, + 0xffff0b0a05040302ULL, 0xffffffffffffffffULL, + 0x0b0a050403020100ULL, 0xffffffffffffffffULL, + 0xffffffff0b0a0706ULL, 0xffffffffffffffffULL, + 0xffff0b0a07060100ULL, 0xffffffffffffffffULL, + 0xffff0b0a07060302ULL, 0xffffffffffffffffULL, + 0x0b0a070603020100ULL, 0xffffffffffffffffULL, + 0xffff0b0a07060504ULL, 0xffffffffffffffffULL, + 0x0b0a070605040100ULL, 0xffffffffffffffffULL, + 0x0b0a070605040302ULL, 0xffffffffffffffffULL, + 0x0706050403020100ULL, 0xffffffffffff0b0aULL, + 0xffffffff0b0a0908ULL, 0xffffffffffffffffULL, + 0xffff0b0a09080100ULL, 0xffffffffffffffffULL, + 0xffff0b0a09080302ULL, 0xffffffffffffffffULL, + 0x0b0a090803020100ULL, 0xffffffffffffffffULL, + 0xffff0b0a09080504ULL, 0xffffffffffffffffULL, + 0x0b0a090805040100ULL, 0xffffffffffffffffULL, + 0x0b0a090805040302ULL, 0xffffffffffffffffULL, + 0x0908050403020100ULL, 0xffffffffffff0b0aULL, + 0xffff0b0a09080706ULL, 0xffffffffffffffffULL, + 0x0b0a090807060100ULL, 0xffffffffffffffffULL, + 0x0b0a090807060302ULL, 0xffffffffffffffffULL, + 0x0908070603020100ULL, 0xffffffffffff0b0aULL, + 0x0b0a090807060504ULL, 0xffffffffffffffffULL, + 0x0908070605040100ULL, 0xffffffffffff0b0aULL, + 0x0908070605040302ULL, 0xffffffffffff0b0aULL, + 0x0706050403020100ULL, 0xffffffff0b0a0908ULL, + 0xffffffffffff0d0cULL, 0xffffffffffffffffULL, + 0xffffffff0d0c0100ULL, 0xffffffffffffffffULL, + 0xffffffff0d0c0302ULL, 0xffffffffffffffffULL, + 0xffff0d0c03020100ULL, 0xffffffffffffffffULL, + 0xffffffff0d0c0504ULL, 0xffffffffffffffffULL, + 0xffff0d0c05040100ULL, 0xffffffffffffffffULL, + 0xffff0d0c05040302ULL, 0xffffffffffffffffULL, + 0x0d0c050403020100ULL, 0xffffffffffffffffULL, + 0xffffffff0d0c0706ULL, 0xffffffffffffffffULL, + 0xffff0d0c07060100ULL, 0xffffffffffffffffULL, + 0xffff0d0c07060302ULL, 0xffffffffffffffffULL, + 0x0d0c070603020100ULL, 0xffffffffffffffffULL, + 0xffff0d0c07060504ULL, 0xffffffffffffffffULL, + 0x0d0c070605040100ULL, 0xffffffffffffffffULL, + 0x0d0c070605040302ULL, 0xffffffffffffffffULL, + 0x0706050403020100ULL, 0xffffffffffff0d0cULL, + 0xffffffff0d0c0908ULL, 0xffffffffffffffffULL, + 0xffff0d0c09080100ULL, 0xffffffffffffffffULL, + 0xffff0d0c09080302ULL, 0xffffffffffffffffULL, + 0x0d0c090803020100ULL, 0xffffffffffffffffULL, + 0xffff0d0c09080504ULL, 0xffffffffffffffffULL, + 0x0d0c090805040100ULL, 0xffffffffffffffffULL, + 0x0d0c090805040302ULL, 0xffffffffffffffffULL, + 0x0908050403020100ULL, 0xffffffffffff0d0cULL, + 0xffff0d0c09080706ULL, 0xffffffffffffffffULL, + 0x0d0c090807060100ULL, 0xffffffffffffffffULL, + 0x0d0c090807060302ULL, 0xffffffffffffffffULL, + 0x0908070603020100ULL, 0xffffffffffff0d0cULL, + 0x0d0c090807060504ULL, 0xffffffffffffffffULL, + 0x0908070605040100ULL, 0xffffffffffff0d0cULL, + 0x0908070605040302ULL, 0xffffffffffff0d0cULL, + 0x0706050403020100ULL, 0xffffffff0d0c0908ULL, + 0xffffffff0d0c0b0aULL, 0xffffffffffffffffULL, + 0xffff0d0c0b0a0100ULL, 0xffffffffffffffffULL, + 0xffff0d0c0b0a0302ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a03020100ULL, 0xffffffffffffffffULL, + 0xffff0d0c0b0a0504ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a05040100ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a05040302ULL, 0xffffffffffffffffULL, + 0x0b0a050403020100ULL, 0xffffffffffff0d0cULL, + 0xffff0d0c0b0a0706ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a07060100ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a07060302ULL, 0xffffffffffffffffULL, + 0x0b0a070603020100ULL, 0xffffffffffff0d0cULL, + 0x0d0c0b0a07060504ULL, 0xffffffffffffffffULL, + 0x0b0a070605040100ULL, 0xffffffffffff0d0cULL, + 0x0b0a070605040302ULL, 0xffffffffffff0d0cULL, + 0x0706050403020100ULL, 0xffffffff0d0c0b0aULL, + 0xffff0d0c0b0a0908ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a09080100ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a09080302ULL, 0xffffffffffffffffULL, + 0x0b0a090803020100ULL, 0xffffffffffff0d0cULL, + 0x0d0c0b0a09080504ULL, 0xffffffffffffffffULL, + 0x0b0a090805040100ULL, 0xffffffffffff0d0cULL, + 0x0b0a090805040302ULL, 0xffffffffffff0d0cULL, + 0x0908050403020100ULL, 0xffffffff0d0c0b0aULL, + 0x0d0c0b0a09080706ULL, 0xffffffffffffffffULL, + 0x0b0a090807060100ULL, 0xffffffffffff0d0cULL, + 0x0b0a090807060302ULL, 0xffffffffffff0d0cULL, + 0x0908070603020100ULL, 0xffffffff0d0c0b0aULL, + 0x0b0a090807060504ULL, 0xffffffffffff0d0cULL, + 0x0908070605040100ULL, 0xffffffff0d0c0b0aULL, + 0x0908070605040302ULL, 0xffffffff0d0c0b0aULL, + 0x0706050403020100ULL, 0xffff0d0c0b0a0908ULL, + 0xffffffffffff0f0eULL, 0xffffffffffffffffULL, + 0xffffffff0f0e0100ULL, 0xffffffffffffffffULL, + 0xffffffff0f0e0302ULL, 0xffffffffffffffffULL, + 0xffff0f0e03020100ULL, 0xffffffffffffffffULL, + 0xffffffff0f0e0504ULL, 0xffffffffffffffffULL, + 0xffff0f0e05040100ULL, 0xffffffffffffffffULL, + 0xffff0f0e05040302ULL, 0xffffffffffffffffULL, + 0x0f0e050403020100ULL, 0xffffffffffffffffULL, + 0xffffffff0f0e0706ULL, 0xffffffffffffffffULL, + 0xffff0f0e07060100ULL, 0xffffffffffffffffULL, + 0xffff0f0e07060302ULL, 0xffffffffffffffffULL, + 0x0f0e070603020100ULL, 0xffffffffffffffffULL, + 0xffff0f0e07060504ULL, 0xffffffffffffffffULL, + 0x0f0e070605040100ULL, 0xffffffffffffffffULL, + 0x0f0e070605040302ULL, 0xffffffffffffffffULL, + 0x0706050403020100ULL, 0xffffffffffff0f0eULL, + 0xffffffff0f0e0908ULL, 0xffffffffffffffffULL, + 0xffff0f0e09080100ULL, 0xffffffffffffffffULL, + 0xffff0f0e09080302ULL, 0xffffffffffffffffULL, + 0x0f0e090803020100ULL, 0xffffffffffffffffULL, + 0xffff0f0e09080504ULL, 0xffffffffffffffffULL, + 0x0f0e090805040100ULL, 0xffffffffffffffffULL, + 0x0f0e090805040302ULL, 0xffffffffffffffffULL, + 0x0908050403020100ULL, 0xffffffffffff0f0eULL, + 0xffff0f0e09080706ULL, 0xffffffffffffffffULL, + 0x0f0e090807060100ULL, 0xffffffffffffffffULL, + 0x0f0e090807060302ULL, 0xffffffffffffffffULL, + 0x0908070603020100ULL, 0xffffffffffff0f0eULL, + 0x0f0e090807060504ULL, 0xffffffffffffffffULL, + 0x0908070605040100ULL, 0xffffffffffff0f0eULL, + 0x0908070605040302ULL, 0xffffffffffff0f0eULL, + 0x0706050403020100ULL, 0xffffffff0f0e0908ULL, + 0xffffffff0f0e0b0aULL, 0xffffffffffffffffULL, + 0xffff0f0e0b0a0100ULL, 0xffffffffffffffffULL, + 0xffff0f0e0b0a0302ULL, 0xffffffffffffffffULL, + 0x0f0e0b0a03020100ULL, 0xffffffffffffffffULL, + 0xffff0f0e0b0a0504ULL, 0xffffffffffffffffULL, + 0x0f0e0b0a05040100ULL, 0xffffffffffffffffULL, + 0x0f0e0b0a05040302ULL, 0xffffffffffffffffULL, + 0x0b0a050403020100ULL, 0xffffffffffff0f0eULL, + 0xffff0f0e0b0a0706ULL, 0xffffffffffffffffULL, + 0x0f0e0b0a07060100ULL, 0xffffffffffffffffULL, + 0x0f0e0b0a07060302ULL, 0xffffffffffffffffULL, + 0x0b0a070603020100ULL, 0xffffffffffff0f0eULL, + 0x0f0e0b0a07060504ULL, 0xffffffffffffffffULL, + 0x0b0a070605040100ULL, 0xffffffffffff0f0eULL, + 0x0b0a070605040302ULL, 0xffffffffffff0f0eULL, + 0x0706050403020100ULL, 0xffffffff0f0e0b0aULL, + 0xffff0f0e0b0a0908ULL, 0xffffffffffffffffULL, + 0x0f0e0b0a09080100ULL, 0xffffffffffffffffULL, + 0x0f0e0b0a09080302ULL, 0xffffffffffffffffULL, + 0x0b0a090803020100ULL, 0xffffffffffff0f0eULL, + 0x0f0e0b0a09080504ULL, 0xffffffffffffffffULL, + 0x0b0a090805040100ULL, 0xffffffffffff0f0eULL, + 0x0b0a090805040302ULL, 0xffffffffffff0f0eULL, + 0x0908050403020100ULL, 0xffffffff0f0e0b0aULL, + 0x0f0e0b0a09080706ULL, 0xffffffffffffffffULL, + 0x0b0a090807060100ULL, 0xffffffffffff0f0eULL, + 0x0b0a090807060302ULL, 0xffffffffffff0f0eULL, + 0x0908070603020100ULL, 0xffffffff0f0e0b0aULL, + 0x0b0a090807060504ULL, 0xffffffffffff0f0eULL, + 0x0908070605040100ULL, 0xffffffff0f0e0b0aULL, + 0x0908070605040302ULL, 0xffffffff0f0e0b0aULL, + 0x0706050403020100ULL, 0xffff0f0e0b0a0908ULL, + 0xffffffff0f0e0d0cULL, 0xffffffffffffffffULL, + 0xffff0f0e0d0c0100ULL, 0xffffffffffffffffULL, + 0xffff0f0e0d0c0302ULL, 0xffffffffffffffffULL, + 0x0f0e0d0c03020100ULL, 0xffffffffffffffffULL, + 0xffff0f0e0d0c0504ULL, 0xffffffffffffffffULL, + 0x0f0e0d0c05040100ULL, 0xffffffffffffffffULL, + 0x0f0e0d0c05040302ULL, 0xffffffffffffffffULL, + 0x0d0c050403020100ULL, 0xffffffffffff0f0eULL, + 0xffff0f0e0d0c0706ULL, 0xffffffffffffffffULL, + 0x0f0e0d0c07060100ULL, 0xffffffffffffffffULL, + 0x0f0e0d0c07060302ULL, 0xffffffffffffffffULL, + 0x0d0c070603020100ULL, 0xffffffffffff0f0eULL, + 0x0f0e0d0c07060504ULL, 0xffffffffffffffffULL, + 0x0d0c070605040100ULL, 0xffffffffffff0f0eULL, + 0x0d0c070605040302ULL, 0xffffffffffff0f0eULL, + 0x0706050403020100ULL, 0xffffffff0f0e0d0cULL, + 0xffff0f0e0d0c0908ULL, 0xffffffffffffffffULL, + 0x0f0e0d0c09080100ULL, 0xffffffffffffffffULL, + 0x0f0e0d0c09080302ULL, 0xffffffffffffffffULL, + 0x0d0c090803020100ULL, 0xffffffffffff0f0eULL, + 0x0f0e0d0c09080504ULL, 0xffffffffffffffffULL, + 0x0d0c090805040100ULL, 0xffffffffffff0f0eULL, + 0x0d0c090805040302ULL, 0xffffffffffff0f0eULL, + 0x0908050403020100ULL, 0xffffffff0f0e0d0cULL, + 0x0f0e0d0c09080706ULL, 0xffffffffffffffffULL, + 0x0d0c090807060100ULL, 0xffffffffffff0f0eULL, + 0x0d0c090807060302ULL, 0xffffffffffff0f0eULL, + 0x0908070603020100ULL, 0xffffffff0f0e0d0cULL, + 0x0d0c090807060504ULL, 0xffffffffffff0f0eULL, + 0x0908070605040100ULL, 0xffffffff0f0e0d0cULL, + 0x0908070605040302ULL, 0xffffffff0f0e0d0cULL, + 0x0706050403020100ULL, 0xffff0f0e0d0c0908ULL, + 0xffff0f0e0d0c0b0aULL, 0xffffffffffffffffULL, + 0x0f0e0d0c0b0a0100ULL, 0xffffffffffffffffULL, + 0x0f0e0d0c0b0a0302ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a03020100ULL, 0xffffffffffff0f0eULL, + 0x0f0e0d0c0b0a0504ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a05040100ULL, 0xffffffffffff0f0eULL, + 0x0d0c0b0a05040302ULL, 0xffffffffffff0f0eULL, + 0x0b0a050403020100ULL, 0xffffffff0f0e0d0cULL, + 0x0f0e0d0c0b0a0706ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a07060100ULL, 0xffffffffffff0f0eULL, + 0x0d0c0b0a07060302ULL, 0xffffffffffff0f0eULL, + 0x0b0a070603020100ULL, 0xffffffff0f0e0d0cULL, + 0x0d0c0b0a07060504ULL, 0xffffffffffff0f0eULL, + 0x0b0a070605040100ULL, 0xffffffff0f0e0d0cULL, + 0x0b0a070605040302ULL, 0xffffffff0f0e0d0cULL, + 0x0706050403020100ULL, 0xffff0f0e0d0c0b0aULL, + 0x0f0e0d0c0b0a0908ULL, 0xffffffffffffffffULL, + 0x0d0c0b0a09080100ULL, 0xffffffffffff0f0eULL, + 0x0d0c0b0a09080302ULL, 0xffffffffffff0f0eULL, + 0x0b0a090803020100ULL, 0xffffffff0f0e0d0cULL, + 0x0d0c0b0a09080504ULL, 0xffffffffffff0f0eULL, + 0x0b0a090805040100ULL, 0xffffffff0f0e0d0cULL, + 0x0b0a090805040302ULL, 0xffffffff0f0e0d0cULL, + 0x0908050403020100ULL, 0xffff0f0e0d0c0b0aULL, + 0x0d0c0b0a09080706ULL, 0xffffffffffff0f0eULL, + 0x0b0a090807060100ULL, 0xffffffff0f0e0d0cULL, + 0x0b0a090807060302ULL, 0xffffffff0f0e0d0cULL, + 0x0908070603020100ULL, 0xffff0f0e0d0c0b0aULL, + 0x0b0a090807060504ULL, 0xffffffff0f0e0d0cULL, + 0x0908070605040100ULL, 0xffff0f0e0d0c0b0aULL, + 0x0908070605040302ULL, 0xffff0f0e0d0c0b0aULL, + 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL, +}; + +XALIGNED(16) static const word16 L_mldsa_extract_coeffs_eta2_mask_nibbles[] + WC_X64I_UNUSED = { + 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, + 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, +}; + +XALIGNED(16) static const word16 L_mldsa_extract_coeffs_eta2_mul[] + WC_X64I_UNUSED = { + 0x3340, 0x3340, 0x3340, 0x3340, 0x3340, 0x3340, 0x3340, 0x3340, + 0x3340, 0x3340, 0x3340, 0x3340, 0x3340, 0x3340, 0x3340, 0x3340, +}; + +XALIGNED(16) static const word16 L_mldsa_extract_coeffs_eta2_five[] + WC_X64I_UNUSED = { + 0x0005, 0x0005, 0x0005, 0x0005, 0x0005, 0x0005, 0x0005, 0x0005, + 0x0005, 0x0005, 0x0005, 0x0005, 0x0005, 0x0005, 0x0005, 0x0005, +}; + +XALIGNED(16) static const word16 L_mldsa_extract_coeffs_eta2_two[] + WC_X64I_UNUSED = { + 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, + 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, +}; + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta2_nibble_table[] + WC_X64I_UNUSED = { + 0x00000002, 0x00000001, 0x00000000, 0xffffffff, + 0xfffffffe, 0x00000002, 0x00000001, 0x00000000, + 0xffffffff, 0xfffffffe, 0x00000002, 0x00000001, + 0x00000000, 0xffffffff, 0xfffffffe, 0x00000000, +}; + +WC_X64I_TARGET("avx2,bmi") +WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta2_avx2(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt) +{ + word64 rdi, rsi, rdx, rcx, r13, r8, rax = 0, r11 = 0, r12 = 0, r10 = 0, + rbx = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), y6, y7, + y8, y9; + unsigned int ac1; + unsigned int ac2; + unsigned char cf; + + rdi = (word64)(size_t)z; + rsi = (word64)(word32)zLen; + rdx = (word64)(size_t)s; + rcx = (word64)(size_t)cnt; + + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_extract_coeffs_eta2_mask_nibbles, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_extract_coeffs_eta2_mul, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_extract_coeffs_eta2_five, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_extract_coeffs_eta2_two, 0)); + r13 = (word64)((word64)(size_t)L_mldsa_shufb_rej_idx); + r8 = (word32)(WC_L32(rcx, 0)); + if (((word32)r8) != (0)) { + goto L_mldsa_extract_coeffs_eta2_less_than_256; + } + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 0))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 8))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, rbx)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r11)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y2 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y1, y7); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_sub_epi16(y9, y0); + y1 = _mm256_sub_epi16(y9, y1); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 16))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 24))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, rbx)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r11)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y2 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y1, y7); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_sub_epi16(y9, y0); + y1 = _mm256_sub_epi16(y9, y1); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 32))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 40))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, rbx)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r11)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y2 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y1, y7); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_sub_epi16(y9, y0); + y1 = _mm256_sub_epi16(y9, y1); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 48))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 56))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, rbx)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r11)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y2 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y1, y7); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_sub_epi16(y9, y0); + y1 = _mm256_sub_epi16(y9, y1); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 64))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 72))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, rbx)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r11)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y2 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y1, y7); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_sub_epi16(y9, y0); + y1 = _mm256_sub_epi16(y9, y1); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 80))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 88))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, rbx)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r11)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y2 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y1, y7); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_sub_epi16(y9, y0); + y1 = _mm256_sub_epi16(y9, y1); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 96))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 104))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, rbx)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r11)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y2 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y1, y7); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_sub_epi16(y9, y0); + y1 = _mm256_sub_epi16(y9, y1); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 112))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 120))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r10)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, rbx)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r11)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y2 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y1, y7); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_sub_epi16(y9, y0); + y1 = _mm256_sub_epi16(y9, y1); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + rsi = (word32)((word32)rsi - 0x80); + rdi = (word64)(rdi + 0x80); +L_mldsa_extract_coeffs_eta2_less_than_256: + if ((sword32)((word32)r8) > (sword32)(0xf0)) { + goto L_mldsa_extract_coeffs_eta2_less_than_ymm; + } +L_mldsa_extract_coeffs_eta2_start_one_ymm: + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 0))); + y2 = _mm256_slli_epi32(y0, 12); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_and_si256(y0, y6); + y2 = _mm256_cmpgt_epi16(y6, y0); + y2 = _mm256_packs_epi16(y2, y2); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + r11 = (word64)(r11 >> 16); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + r11 = (word32)((word32)r11 << 4); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r10)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r13, r11)); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + y2 = _mm256_mulhi_epi16(y0, y7); + y2 = _mm256_mullo_epi16(y2, y8); + y0 = _mm256_sub_epi16(y9, y0); + y0 = _mm256_add_epi16(y0, y2); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + rsi = (word32)((word32)rsi - 8); + rdi = (word64)(rdi + 8); + if ((sword32)((word32)rsi) < (sword32)(8)) { + goto L_mldsa_extract_coeffs_eta2_less_than_ymm; + } + if ((sword32)((word32)r8) <= (sword32)(0xf0)) { + goto L_mldsa_extract_coeffs_eta2_start_one_ymm; + } +L_mldsa_extract_coeffs_eta2_less_than_ymm: + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta2_done; + } +L_mldsa_extract_coeffs_eta2_start_byte: + r13 = (word64)((word64)(size_t)L_mldsa_extract_coeffs_eta2_nibble_table); + if (((word32)rsi) == (0)) { + goto L_mldsa_extract_coeffs_eta2_done; + } + rbx = (word32)((word32)WC_L8(rdi, 0)); + rdi = (word64)(rdi + 1); + rsi = (word32)((word32)rsi - 1); + rax = (word32)((word32)rbx); + rax = (word32)((word32)rax >> 4); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)((byte)rbx & 0xf) & 0xff); + r11 = (word64)(0); + cf = _addcarry_u32((unsigned char)(((byte)rbx) < (0xf)), (word32)( + word32)r11, (word32)0, &ac1); + r11 = (word32)ac1; + r12 = (word32)(WC_L32(r13, rbx * 4)); + WC_S32(rdx, 0) = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)r11); + r11 = (word32)((word32)r11 << 2); + rdx = (word64)(rdx + r11); + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta2_done; + } + r11 = (word64)(0); + cf = _addcarry_u32((unsigned char)(((byte)rax) < (0xf)), (word32)( + word32)r11, (word32)0, &ac2); + r11 = (word32)ac2; + r12 = (word32)(WC_L32(r13, rax * 4)); + WC_S32(rdx, 0) = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)r11); + r11 = (word32)((word32)r11 << 2); + rdx = (word64)(rdx + r11); + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta2_done; + } + goto L_mldsa_extract_coeffs_eta2_start_byte; +L_mldsa_extract_coeffs_eta2_done: + WC_S32(rcx, 0) = (word32)((word32)r8); + (void)cf; +} + +XALIGNED(16) static const word16 L_mldsa_extract_coeffs_eta4_mask_nibbles[] + WC_X64I_UNUSED = { + 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, + 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, +}; + +XALIGNED(16) static const word16 L_mldsa_extract_coeffs_eta4_nine[] + WC_X64I_UNUSED = { + 0x0009, 0x0009, 0x0009, 0x0009, 0x0009, 0x0009, 0x0009, 0x0009, + 0x0009, 0x0009, 0x0009, 0x0009, 0x0009, 0x0009, 0x0009, 0x0009, +}; + +XALIGNED(16) static const word16 L_mldsa_extract_coeffs_eta4_four[] + WC_X64I_UNUSED = { + 0x0004, 0x0004, 0x0004, 0x0004, 0x0004, 0x0004, 0x0004, 0x0004, + 0x0004, 0x0004, 0x0004, 0x0004, 0x0004, 0x0004, 0x0004, 0x0004, +}; + +WC_X64I_TARGET("avx2,bmi") +WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta4_avx2(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt) +{ + word64 rdi, rsi, rdx, rcx, r13, r8, rax = 0, r11 = 0, r12 = 0, r10 = 0, + rbx = 0; + __m128i x4 = _mm_setzero_si128(), x5 = _mm_setzero_si128(); + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), y6, y7, + y8; + unsigned int ac1; + unsigned int ac2; + unsigned char cf; + + rdi = (word64)(size_t)z; + rsi = (word64)(word32)zLen; + rdx = (word64)(size_t)s; + rcx = (word64)(size_t)cnt; + + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_extract_coeffs_eta4_mask_nibbles, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_extract_coeffs_eta4_nine, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_extract_coeffs_eta4_four, 0)); + r13 = (word64)((word64)(size_t)L_mldsa_shufb_rej_idx); + r8 = (word32)(WC_L32(rcx, 0)); + if (((word32)r8) != (0)) { + goto L_mldsa_extract_coeffs_eta4_less_than_256; + } + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 0))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 8))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y7, y0); + y3 = _mm256_cmpgt_epi16(y7, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + r10))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r11)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, x4, 1); + y3 = _mm256_inserti128_si256(y3, x5, 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y8, y1); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 16))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 24))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y7, y0); + y3 = _mm256_cmpgt_epi16(y7, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + r10))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r11)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, x4, 1); + y3 = _mm256_inserti128_si256(y3, x5, 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y8, y1); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 32))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 40))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y7, y0); + y3 = _mm256_cmpgt_epi16(y7, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + r10))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r11)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, x4, 1); + y3 = _mm256_inserti128_si256(y3, x5, 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y8, y1); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 48))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 56))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y7, y0); + y3 = _mm256_cmpgt_epi16(y7, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + r10))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r11)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, x4, 1); + y3 = _mm256_inserti128_si256(y3, x5, 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y8, y1); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 64))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 72))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y7, y0); + y3 = _mm256_cmpgt_epi16(y7, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + r10))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r11)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, x4, 1); + y3 = _mm256_inserti128_si256(y3, x5, 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y8, y1); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 80))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 88))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y7, y0); + y3 = _mm256_cmpgt_epi16(y7, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + r10))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r11)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, x4, 1); + y3 = _mm256_inserti128_si256(y3, x5, 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y8, y1); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 96))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 104))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y7, y0); + y3 = _mm256_cmpgt_epi16(y7, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + r10))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r11)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, x4, 1); + y3 = _mm256_inserti128_si256(y3, x5, 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y8, y1); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 112))); + y1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 120))); + y2 = _mm256_slli_epi32(y0, 12); + y3 = _mm256_slli_epi32(y1, 12); + y0 = _mm256_or_si256(y0, y2); + y1 = _mm256_or_si256(y1, y3); + y0 = _mm256_and_si256(y0, y6); + y1 = _mm256_and_si256(y1, y6); + y2 = _mm256_cmpgt_epi16(y7, y0); + y3 = _mm256_cmpgt_epi16(y7, y1); + y2 = _mm256_packs_epi16(y2, y3); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r12 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + rbx = (word32)((word32)(byte)(rax >> 8)); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + rbx = (word32)((word32)rbx << 4); + r11 = (word32)((word32)r11 << 4); + r12 = (word32)((word32)r12 << 4); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + r10))); + y3 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + rbx))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r11)); + x5 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r12)); + y2 = _mm256_inserti128_si256(y2, x4, 1); + y3 = _mm256_inserti128_si256(y3, x5, 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y8, y1); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y3 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y1 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y3); + rdx = (word64)(rdx + rbx * 4); + r8 = (word32)((word32)r8 + (word32)rbx); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + rdx = (word64)(rdx + r12 * 4); + r8 = (word32)((word32)r8 + (word32)r12); + rsi = (word32)((word32)rsi - 0x80); + rdi = (word64)(rdi + 0x80); +L_mldsa_extract_coeffs_eta4_less_than_256: + if ((sword32)((word32)r8) > (sword32)(0xf0)) { + goto L_mldsa_extract_coeffs_eta4_less_than_ymm; + } +L_mldsa_extract_coeffs_eta4_start_one_ymm: + y0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)WC_PR(rdi, 0))); + y2 = _mm256_slli_epi32(y0, 12); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_and_si256(y0, y6); + y2 = _mm256_cmpgt_epi16(y7, y0); + y2 = _mm256_packs_epi16(y2, y2); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + r11 = (word32)((word32)rax); + r10 = (word32)((word32)(byte)rax); + r11 = (word64)(r11 >> 16); + r11 = (word32)((word32)r11 & 0xff); + r10 = (word32)((word32)r10 << 4); + r11 = (word32)((word32)r11 << 4); + y2 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(r13, + r10))); + x4 = _mm_loadu_si128((const __m128i*)WC_PR(r13, r11)); + y2 = _mm256_inserti128_si256(y2, x4, 1); + y0 = _mm256_shuffle_epi8(y0, y2); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + y0 = _mm256_sub_epi16(y8, y0); + y2 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(y0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y2); + rdx = (word64)(rdx + r10 * 4); + r8 = (word32)((word32)r8 + (word32)r10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + rdx = (word64)(rdx + r11 * 4); + r8 = (word32)((word32)r8 + (word32)r11); + rsi = (word32)((word32)rsi - 8); + rdi = (word64)(rdi + 8); + if ((sword32)((word32)rsi) < (sword32)(8)) { + goto L_mldsa_extract_coeffs_eta4_less_than_ymm; + } + if ((sword32)((word32)r8) <= (sword32)(0xf0)) { + goto L_mldsa_extract_coeffs_eta4_start_one_ymm; + } +L_mldsa_extract_coeffs_eta4_less_than_ymm: + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta4_done; + } +L_mldsa_extract_coeffs_eta4_start_byte: + if (((word32)rsi) == (0)) { + goto L_mldsa_extract_coeffs_eta4_done; + } + rbx = (word32)((word32)WC_L8(rdi, 0)); + rdi = (word64)(rdi + 1); + rsi = (word32)((word32)rsi - 1); + rax = (word32)((word32)rbx); + rax = (word32)((word32)rax >> 4); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)((byte)rbx & 0xf) & 0xff); + r11 = (word64)(0); + r12 = (word64)(4); + cf = _addcarry_u32((unsigned char)(((byte)rbx) < (9)), (word32)(word32)r11, + (word32)0, &ac1); + r11 = (word32)ac1; + r12 = (word32)((word32)r12 - (word32)rbx); + WC_S32(rdx, 0) = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)r11); + r11 = (word32)((word32)r11 << 2); + rdx = (word64)(rdx + r11); + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta4_done; + } + r11 = (word64)(0); + r12 = (word64)(4); + cf = _addcarry_u32((unsigned char)(((byte)rax) < (9)), (word32)(word32)r11, + (word32)0, &ac2); + r11 = (word32)ac2; + r12 = (word32)((word32)r12 - (word32)rax); + WC_S32(rdx, 0) = (word32)((word32)r12); + r8 = (word32)((word32)r8 + (word32)r11); + r11 = (word32)((word32)r11 << 2); + rdx = (word64)(rdx + r11); + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta4_done; + } + goto L_mldsa_extract_coeffs_eta4_start_byte; +L_mldsa_extract_coeffs_eta4_done: + WC_S32(rcx, 0) = (word32)((word32)r8); + (void)cf; +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_redistribute_21_rand_avx2(word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y12 = _mm256_unpacklo_epi64(y0, y1); + y13 = _mm256_unpackhi_epi64(y0, y1); + y14 = _mm256_unpacklo_epi64(y2, y3); + y15 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y12, y14, 0x20); + y1 = _mm256_permute2x128_si256(y13, y15, 0x20); + y2 = _mm256_permute2x128_si256(y12, y14, 0x31); + y3 = _mm256_permute2x128_si256(y13, y15, 0x31); + y12 = _mm256_unpacklo_epi64(y4, y5); + y13 = _mm256_unpackhi_epi64(y4, y5); + y14 = _mm256_unpacklo_epi64(y6, y7); + y15 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y12, y14, 0x20); + y5 = _mm256_permute2x128_si256(y13, y15, 0x20); + y6 = _mm256_permute2x128_si256(y12, y14, 0x31); + y7 = _mm256_permute2x128_si256(y13, y15, 0x31); + y12 = _mm256_unpacklo_epi64(y8, y9); + y13 = _mm256_unpackhi_epi64(y8, y9); + y14 = _mm256_unpacklo_epi64(y10, y11); + y15 = _mm256_unpackhi_epi64(y10, y11); + y8 = _mm256_permute2x128_si256(y12, y14, 0x20); + y9 = _mm256_permute2x128_si256(y13, y15, 0x20); + y10 = _mm256_permute2x128_si256(y12, y14, 0x31); + y11 = _mm256_permute2x128_si256(y13, y15, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y10); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y7); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + rax = (word64)(WC_L64(rdi, 640)); + r9 = (word64)(WC_L64(rdi, 648)); + r10 = (word64)(WC_L64(rdi, 656)); + r11 = (word64)(WC_L64(rdi, 664)); + y12 = _mm256_unpacklo_epi64(y0, y1); + y13 = _mm256_unpackhi_epi64(y0, y1); + y14 = _mm256_unpacklo_epi64(y2, y3); + y15 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y12, y14, 0x20); + y1 = _mm256_permute2x128_si256(y13, y15, 0x20); + y2 = _mm256_permute2x128_si256(y12, y14, 0x31); + y3 = _mm256_permute2x128_si256(y13, y15, 0x31); + y12 = _mm256_unpacklo_epi64(y4, y5); + y13 = _mm256_unpackhi_epi64(y4, y5); + y14 = _mm256_unpacklo_epi64(y6, y7); + y15 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y12, y14, 0x20); + y5 = _mm256_permute2x128_si256(y13, y15, 0x20); + y6 = _mm256_permute2x128_si256(y12, y14, 0x31); + y7 = _mm256_permute2x128_si256(y13, y15, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y4); + WC_S64(rsi, 160) = (word64)(rax); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y5); + WC_S64(rdx, 160) = (word64)(r9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y6); + WC_S64(rcx, 160) = (word64)(r10); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y7); + WC_S64(r8, 160) = (word64)(r11); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_redistribute_17_rand_avx2(word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y9 = _mm256_unpackhi_epi64(y0, y1); + y10 = _mm256_unpacklo_epi64(y2, y3); + y11 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y8, y10, 0x20); + y1 = _mm256_permute2x128_si256(y9, y11, 0x20); + y2 = _mm256_permute2x128_si256(y8, y10, 0x31); + y3 = _mm256_permute2x128_si256(y9, y11, 0x31); + y8 = _mm256_unpacklo_epi64(y4, y5); + y9 = _mm256_unpackhi_epi64(y4, y5); + y10 = _mm256_unpacklo_epi64(y6, y7); + y11 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y8, y10, 0x20); + y5 = _mm256_permute2x128_si256(y9, y11, 0x20); + y6 = _mm256_permute2x128_si256(y8, y10, 0x31); + y7 = _mm256_permute2x128_si256(y9, y11, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + rax = (word64)(WC_L64(rdi, 512)); + r9 = (word64)(WC_L64(rdi, 520)); + r10 = (word64)(WC_L64(rdi, 528)); + r11 = (word64)(WC_L64(rdi, 536)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y9 = _mm256_unpackhi_epi64(y0, y1); + y10 = _mm256_unpacklo_epi64(y2, y3); + y11 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y8, y10, 0x20); + y1 = _mm256_permute2x128_si256(y9, y11, 0x20); + y2 = _mm256_permute2x128_si256(y8, y10, 0x31); + y3 = _mm256_permute2x128_si256(y9, y11, 0x31); + y8 = _mm256_unpacklo_epi64(y4, y5); + y9 = _mm256_unpackhi_epi64(y4, y5); + y10 = _mm256_unpacklo_epi64(y6, y7); + y11 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y8, y10, 0x20); + y5 = _mm256_permute2x128_si256(y9, y11, 0x20); + y6 = _mm256_permute2x128_si256(y8, y10, 0x31); + y7 = _mm256_permute2x128_si256(y9, y11, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y4); + WC_S64(rsi, 128) = (word64)(rax); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y5); + WC_S64(rdx, 128) = (word64)(r9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y6); + WC_S64(rcx, 128) = (word64)(r10); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y7); + WC_S64(r8, 128) = (word64)(r11); +} + +XALIGNED(16) static const word32 L_mldsa_encode_eta_2_avx2_two[] + WC_X64I_UNUSED = { + 0x00000002, 0x00000002, 0x00000002, 0x00000002, + 0x00000002, 0x00000002, 0x00000002, 0x00000002, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_eta_2_avx2_vs_3[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000003, 0x00000006, 0x00000009, + 0x00000004, 0x00000007, 0x0000000a, 0x0000000d, +}; + +XALIGNED(16) static const byte L_mldsa_encode_eta_2_avx2_shuff_3_even[] + WC_X64I_UNUSED = { + 0x00, 0xff, 0x04, 0x05, 0x08, 0xff, 0x0c, 0x0d, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0xff, 0x04, 0x05, 0x08, 0xff, 0x0c, 0x0d, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_eta_2_avx2_shuff_3_odd[] + WC_X64I_UNUSED = { + 0x02, 0xff, 0xff, 0x07, 0x0a, 0x0b, 0xff, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x02, 0xff, 0xff, 0x07, 0x0a, 0x0b, 0xff, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_eta_2_avx2_shuff_6_even[] + WC_X64I_UNUSED = { + 0x00, 0x04, 0x05, 0x08, 0x0c, 0x0d, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x04, + 0x05, 0x08, 0x0c, 0x0d, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_eta_2_avx2_shuff_6_odd[] + WC_X64I_UNUSED = { + 0x02, 0x03, 0x07, 0x0a, 0x0b, 0x0f, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x03, + 0x07, 0x0a, 0x0b, 0x0f, 0xff, 0xff, 0xff, 0xff, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_2_avx2(const sword32* s, byte k, + byte* p) +{ + word64 rdi, rsi, rdx; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), y6, y7, + y8, y9, y10, y11; + + rdi = (word64)(size_t)s; + rsi = (word64)(byte)k; + rdx = (word64)(size_t)p; + + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_encode_eta_2_avx2_two, + 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_avx2_vs_3, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_avx2_shuff_3_even, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_avx2_shuff_3_odd, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_avx2_shuff_6_even, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_avx2_shuff_6_odd, 0)); +L_mldsa_encode_eta_2_avx2_loop: + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y0 = _mm256_sub_epi32(y6, y0); + y1 = _mm256_sub_epi32(y6, y1); + y2 = _mm256_sub_epi32(y6, y2); + y3 = _mm256_sub_epi32(y6, y3); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_packus_epi32(y0, y2); + y1 = _mm256_packus_epi32(y1, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y4 = _mm256_shuffle_epi8(y0, y9); + y5 = _mm256_shuffle_epi8(y1, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y1 = _mm256_permute4x64_epi64(y1, 0xb1); + y0 = _mm256_or_si256(y0, y1); + y4 = _mm256_shuffle_epi8(y0, y11); + y0 = _mm256_shuffle_epi8(y0, y10); + y0 = _mm256_or_si256(y0, y4); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(rdx + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y0 = _mm256_sub_epi32(y6, y0); + y1 = _mm256_sub_epi32(y6, y1); + y2 = _mm256_sub_epi32(y6, y2); + y3 = _mm256_sub_epi32(y6, y3); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_packus_epi32(y0, y2); + y1 = _mm256_packus_epi32(y1, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y4 = _mm256_shuffle_epi8(y0, y9); + y5 = _mm256_shuffle_epi8(y1, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y1 = _mm256_permute4x64_epi64(y1, 0xb1); + y0 = _mm256_or_si256(y0, y1); + y4 = _mm256_shuffle_epi8(y0, y11); + y0 = _mm256_shuffle_epi8(y0, y10); + y0 = _mm256_or_si256(y0, y4); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(rdx + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y0 = _mm256_sub_epi32(y6, y0); + y1 = _mm256_sub_epi32(y6, y1); + y2 = _mm256_sub_epi32(y6, y2); + y3 = _mm256_sub_epi32(y6, y3); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_packus_epi32(y0, y2); + y1 = _mm256_packus_epi32(y1, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y4 = _mm256_shuffle_epi8(y0, y9); + y5 = _mm256_shuffle_epi8(y1, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y1 = _mm256_permute4x64_epi64(y1, 0xb1); + y0 = _mm256_or_si256(y0, y1); + y4 = _mm256_shuffle_epi8(y0, y11); + y0 = _mm256_shuffle_epi8(y0, y10); + y0 = _mm256_or_si256(y0, y4); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(rdx + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y0 = _mm256_sub_epi32(y6, y0); + y1 = _mm256_sub_epi32(y6, y1); + y2 = _mm256_sub_epi32(y6, y2); + y3 = _mm256_sub_epi32(y6, y3); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_packus_epi32(y0, y2); + y1 = _mm256_packus_epi32(y1, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y4 = _mm256_shuffle_epi8(y0, y9); + y5 = _mm256_shuffle_epi8(y1, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y1 = _mm256_permute4x64_epi64(y1, 0xb1); + y0 = _mm256_or_si256(y0, y1); + y4 = _mm256_shuffle_epi8(y0, y11); + y0 = _mm256_shuffle_epi8(y0, y10); + y0 = _mm256_or_si256(y0, y4); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(rdx + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y0 = _mm256_sub_epi32(y6, y0); + y1 = _mm256_sub_epi32(y6, y1); + y2 = _mm256_sub_epi32(y6, y2); + y3 = _mm256_sub_epi32(y6, y3); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_packus_epi32(y0, y2); + y1 = _mm256_packus_epi32(y1, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y4 = _mm256_shuffle_epi8(y0, y9); + y5 = _mm256_shuffle_epi8(y1, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y1 = _mm256_permute4x64_epi64(y1, 0xb1); + y0 = _mm256_or_si256(y0, y1); + y4 = _mm256_shuffle_epi8(y0, y11); + y0 = _mm256_shuffle_epi8(y0, y10); + y0 = _mm256_or_si256(y0, y4); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(rdx + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y0 = _mm256_sub_epi32(y6, y0); + y1 = _mm256_sub_epi32(y6, y1); + y2 = _mm256_sub_epi32(y6, y2); + y3 = _mm256_sub_epi32(y6, y3); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_packus_epi32(y0, y2); + y1 = _mm256_packus_epi32(y1, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y4 = _mm256_shuffle_epi8(y0, y9); + y5 = _mm256_shuffle_epi8(y1, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y1 = _mm256_permute4x64_epi64(y1, 0xb1); + y0 = _mm256_or_si256(y0, y1); + y4 = _mm256_shuffle_epi8(y0, y11); + y0 = _mm256_shuffle_epi8(y0, y10); + y0 = _mm256_or_si256(y0, y4); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(rdx + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y0 = _mm256_sub_epi32(y6, y0); + y1 = _mm256_sub_epi32(y6, y1); + y2 = _mm256_sub_epi32(y6, y2); + y3 = _mm256_sub_epi32(y6, y3); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_packus_epi32(y0, y2); + y1 = _mm256_packus_epi32(y1, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y4 = _mm256_shuffle_epi8(y0, y9); + y5 = _mm256_shuffle_epi8(y1, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y1 = _mm256_permute4x64_epi64(y1, 0xb1); + y0 = _mm256_or_si256(y0, y1); + y4 = _mm256_shuffle_epi8(y0, y11); + y0 = _mm256_shuffle_epi8(y0, y10); + y0 = _mm256_or_si256(y0, y4); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(rdx + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y0 = _mm256_sub_epi32(y6, y0); + y1 = _mm256_sub_epi32(y6, y1); + y2 = _mm256_sub_epi32(y6, y2); + y3 = _mm256_sub_epi32(y6, y3); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_packus_epi32(y0, y2); + y1 = _mm256_packus_epi32(y1, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y4 = _mm256_shuffle_epi8(y0, y9); + y5 = _mm256_shuffle_epi8(y1, y9); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_shuffle_epi8(y1, y8); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y1 = _mm256_permute4x64_epi64(y1, 0xb1); + y0 = _mm256_or_si256(y0, y1); + y4 = _mm256_shuffle_epi8(y0, y11); + y0 = _mm256_shuffle_epi8(y0, y10); + y0 = _mm256_or_si256(y0, y4); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y4); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y0)); + rdx = (word64)(rdx + 0xc); + rdi = (word64)(rdi + 0x400); + rsi = (rsi & ~(word64)0xff) | ((word64)(byte)((byte)rsi - 1) & 0xff); + if (((byte)rsi) != (0)) { + goto L_mldsa_encode_eta_2_avx2_loop; + } +} + +XALIGNED(16) static const word32 L_mldsa_encode_eta_4_avx2_four[] + WC_X64I_UNUSED = { + 0x00000004, 0x00000004, 0x00000004, 0x00000004, + 0x00000004, 0x00000004, 0x00000004, 0x00000004, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_eta_4_avx2_vs_4[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000000, 0x00000004, 0x00000000, 0x00000004, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_4_avx2(const sword32* s, byte* p) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)p; + + y8 = _mm256_setzero_si256(); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_4_avx2_four, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_4_avx2_vs_4, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1024)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1056)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1088)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1120)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1152)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1184)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1216)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1248)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1280)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1312)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1344)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1376)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1408)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1440)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1472)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1504)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1536)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1568)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1600)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1632)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1664)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1696)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1728)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1760)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1792)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1824)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1856)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1888)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1920)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1952)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1984)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2016)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2048)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2080)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2112)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2144)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2176)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2208)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2240)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2272)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2304)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2336)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2368)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2400)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2432)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2464)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2496)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2528)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2560)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2592)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2624)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2656)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2688)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2720)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2752)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2784)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2816)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2848)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2880)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2912)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2944)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2976)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3008)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3040)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3072)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3104)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3136)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3168)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3200)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3232)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3264)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3296)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3328)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3360)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3392)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3424)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3456)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3488)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3520)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3552)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3584)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3616)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3648)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3680)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3712)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3744)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3776)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3808)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3840)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3872)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3904)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3936)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3968)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4000)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4032)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4064)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4096)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4128)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4160)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4192)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4224)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4256)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4288)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4320)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 512), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4352)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4384)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4416)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4448)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4480)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4512)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4544)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4576)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 544), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4608)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4640)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4672)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4704)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4736)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4768)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4800)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4832)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 576), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4864)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4896)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4928)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4960)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4992)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5024)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5056)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5088)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 608), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5120)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5152)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5184)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5216)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5248)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5280)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5312)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5344)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 640), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5376)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5408)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5440)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5472)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5504)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5536)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5568)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5600)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 672), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5632)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5664)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5696)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5728)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5760)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5792)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5824)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5856)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 704), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5888)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5920)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5952)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 5984)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 6016)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 6048)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 6080)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 6112)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + y0 = _mm256_sllv_epi32(y0, y9); + y1 = _mm256_sllv_epi32(y1, y9); + y2 = _mm256_sllv_epi32(y2, y9); + y3 = _mm256_sllv_epi32(y3, y9); + y4 = _mm256_sllv_epi32(y4, y9); + y5 = _mm256_sllv_epi32(y5, y9); + y6 = _mm256_sllv_epi32(y6, y9); + y7 = _mm256_sllv_epi32(y7, y9); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 736), y0); +} + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_avx2_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, + 0x00, 0x01, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_avx2_shuff_1[] + WC_X64I_UNUSED = { + 0x01, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0x02, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_avx2_shuff_2[] + WC_X64I_UNUSED = { + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x02, 0x03, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0x04, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_avx2_shuff_3[] + WC_X64I_UNUSED = { + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x03, 0x04, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0x05, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_avx2_shuff_4[] + WC_X64I_UNUSED = { + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x04, 0x05, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_avx2_shuff_5[] + WC_X64I_UNUSED = { + 0x05, 0xff, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x05, 0x06, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_avx2_shuff_6[] + WC_X64I_UNUSED = { + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x06, 0x07, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_avx2_shuff_7[] + WC_X64I_UNUSED = { + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x07, 0x08, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, 0xff, + 0x09, 0xff, 0xff, 0xff, 0x09, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_2_avx2_two[] + WC_X64I_UNUSED = { + 0x00000002, 0x00000002, 0x00000002, 0x00000002, + 0x00000002, 0x00000002, 0x00000002, 0x00000002, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_2_avx2_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000003, 0x00000006, 0x00000001, + 0x00000004, 0x00000007, 0x00000002, 0x00000005, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_2_avx2_mask[] + WC_X64I_UNUSED = { + 0x00000007, 0x00000007, 0x00000007, 0x00000007, + 0x00000007, 0x00000007, 0x00000007, 0x00000007, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_decode_eta_2_avx2(const byte* p, sword32* s) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)s; + + y4 = _mm256_setzero_si256(); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_shuff_0, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_shuff_1, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_shuff_2, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_shuff_3, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_shuff_4, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_shuff_5, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_shuff_6, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_shuff_7, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_two, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_decode_eta_2_avx2_vs, + 0)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_avx2_mask, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y4); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y3); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y7); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y3); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y10); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y3); + y0 = _mm256_permute4x64_epi64(y0, 0x39); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y5); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y3); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y8); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y3); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y11); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y3); + y0 = _mm256_permute4x64_epi64(y0, 0x39); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y6); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y3); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y9); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y3); + y0 = _mm256_permute4x64_epi64(y0, 0x39); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y4); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y3); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y7); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y3); + y0 = _mm256_permute2x128_si256(y0, y1, 0x20); + y0 = _mm256_permute4x64_epi64(y0, 0x38); + y3 = _mm256_permute4x64_epi64(y0, 0x44); + y3 = _mm256_shuffle_epi8(y3, y10); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y3); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y5); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y3); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y8); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y3); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y11); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y3); + y1 = _mm256_permute4x64_epi64(y1, 0x39); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y6); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y3); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y9); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y3); + y1 = _mm256_permute4x64_epi64(y1, 0x39); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y4); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 512), y3); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y7); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 544), y3); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y10); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 576), y3); + y1 = _mm256_permute4x64_epi64(y1, 0x39); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y5); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 608), y3); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y8); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 640), y3); + y1 = _mm256_permute2x128_si256(y1, y2, 0x20); + y1 = _mm256_permute4x64_epi64(y1, 0x38); + y3 = _mm256_permute4x64_epi64(y1, 0x44); + y3 = _mm256_shuffle_epi8(y3, y11); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 672), y3); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y6); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 704), y3); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y9); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 736), y3); + y2 = _mm256_permute4x64_epi64(y2, 0x39); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y4); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 768), y3); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y7); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 800), y3); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y10); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 832), y3); + y2 = _mm256_permute4x64_epi64(y2, 0x39); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y5); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 864), y3); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y8); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 896), y3); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y11); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 928), y3); + y2 = _mm256_permute4x64_epi64(y2, 0x39); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y6); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 960), y3); + y3 = _mm256_permute4x64_epi64(y2, 0x44); + y3 = _mm256_shuffle_epi8(y3, y9); + y3 = _mm256_srlv_epi32(y3, y13); + y3 = _mm256_and_si256(y3, y14); + y3 = _mm256_sub_epi32(y12, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 992), y3); +} + +XALIGNED(16) static const byte L_mldsa_decode_eta_4_avx2_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, + 0x00, 0x01, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_4_avx2_shuff_1[] + WC_X64I_UNUSED = { + 0x01, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0x02, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_4_avx2_shuff_2[] + WC_X64I_UNUSED = { + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x02, 0x03, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0x04, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_4_avx2_shuff_3[] + WC_X64I_UNUSED = { + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x03, 0x04, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0x05, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_4_avx2_shuff_4[] + WC_X64I_UNUSED = { + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x04, 0x05, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_4_avx2_shuff_5[] + WC_X64I_UNUSED = { + 0x05, 0xff, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x05, 0x06, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_4_avx2_shuff_6[] + WC_X64I_UNUSED = { + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x06, 0x07, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_4_avx2_shuff_7[] + WC_X64I_UNUSED = { + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x07, 0x08, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, 0xff, + 0x09, 0xff, 0xff, 0xff, 0x09, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_4_avx2_four[] + WC_X64I_UNUSED = { + 0x00000004, 0x00000004, 0x00000004, 0x00000004, + 0x00000004, 0x00000004, 0x00000004, 0x00000004, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_4_avx2_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000008, 0x0000000c, + 0x00000010, 0x00000014, 0x00000018, 0x0000001c, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_4_avx2_mask[] + WC_X64I_UNUSED = { + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_decode_eta_4_avx2(const byte* p, sword32* s) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)s; + + y8 = _mm256_setzero_si256(); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_4_avx2_four, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_decode_eta_4_avx2_vs, + 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_4_avx2_mask, 0)); + y0 = _mm256_set1_epi32((int)WC_L32(rdi, 0)); + y1 = _mm256_set1_epi32((int)WC_L32(rdi, 4)); + y2 = _mm256_set1_epi32((int)WC_L32(rdi, 8)); + y3 = _mm256_set1_epi32((int)WC_L32(rdi, 12)); + y4 = _mm256_set1_epi32((int)WC_L32(rdi, 16)); + y5 = _mm256_set1_epi32((int)WC_L32(rdi, 20)); + y6 = _mm256_set1_epi32((int)WC_L32(rdi, 24)); + y7 = _mm256_set1_epi32((int)WC_L32(rdi, 28)); + y0 = _mm256_srlv_epi32(y0, y9); + y1 = _mm256_srlv_epi32(y1, y9); + y2 = _mm256_srlv_epi32(y2, y9); + y3 = _mm256_srlv_epi32(y3, y9); + y4 = _mm256_srlv_epi32(y4, y9); + y5 = _mm256_srlv_epi32(y5, y9); + y6 = _mm256_srlv_epi32(y6, y9); + y7 = _mm256_srlv_epi32(y7, y9); + y0 = _mm256_and_si256(y0, y10); + y1 = _mm256_and_si256(y1, y10); + y2 = _mm256_and_si256(y2, y10); + y3 = _mm256_and_si256(y3, y10); + y4 = _mm256_and_si256(y4, y10); + y5 = _mm256_and_si256(y5, y10); + y6 = _mm256_and_si256(y6, y10); + y7 = _mm256_and_si256(y7, y10); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y7); + y0 = _mm256_set1_epi32((int)WC_L32(rdi, 32)); + y1 = _mm256_set1_epi32((int)WC_L32(rdi, 36)); + y2 = _mm256_set1_epi32((int)WC_L32(rdi, 40)); + y3 = _mm256_set1_epi32((int)WC_L32(rdi, 44)); + y4 = _mm256_set1_epi32((int)WC_L32(rdi, 48)); + y5 = _mm256_set1_epi32((int)WC_L32(rdi, 52)); + y6 = _mm256_set1_epi32((int)WC_L32(rdi, 56)); + y7 = _mm256_set1_epi32((int)WC_L32(rdi, 60)); + y0 = _mm256_srlv_epi32(y0, y9); + y1 = _mm256_srlv_epi32(y1, y9); + y2 = _mm256_srlv_epi32(y2, y9); + y3 = _mm256_srlv_epi32(y3, y9); + y4 = _mm256_srlv_epi32(y4, y9); + y5 = _mm256_srlv_epi32(y5, y9); + y6 = _mm256_srlv_epi32(y6, y9); + y7 = _mm256_srlv_epi32(y7, y9); + y0 = _mm256_and_si256(y0, y10); + y1 = _mm256_and_si256(y1, y10); + y2 = _mm256_and_si256(y2, y10); + y3 = _mm256_and_si256(y3, y10); + y4 = _mm256_and_si256(y4, y10); + y5 = _mm256_and_si256(y5, y10); + y6 = _mm256_and_si256(y6, y10); + y7 = _mm256_and_si256(y7, y10); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y7); + y0 = _mm256_set1_epi32((int)WC_L32(rdi, 64)); + y1 = _mm256_set1_epi32((int)WC_L32(rdi, 68)); + y2 = _mm256_set1_epi32((int)WC_L32(rdi, 72)); + y3 = _mm256_set1_epi32((int)WC_L32(rdi, 76)); + y4 = _mm256_set1_epi32((int)WC_L32(rdi, 80)); + y5 = _mm256_set1_epi32((int)WC_L32(rdi, 84)); + y6 = _mm256_set1_epi32((int)WC_L32(rdi, 88)); + y7 = _mm256_set1_epi32((int)WC_L32(rdi, 92)); + y0 = _mm256_srlv_epi32(y0, y9); + y1 = _mm256_srlv_epi32(y1, y9); + y2 = _mm256_srlv_epi32(y2, y9); + y3 = _mm256_srlv_epi32(y3, y9); + y4 = _mm256_srlv_epi32(y4, y9); + y5 = _mm256_srlv_epi32(y5, y9); + y6 = _mm256_srlv_epi32(y6, y9); + y7 = _mm256_srlv_epi32(y7, y9); + y0 = _mm256_and_si256(y0, y10); + y1 = _mm256_and_si256(y1, y10); + y2 = _mm256_and_si256(y2, y10); + y3 = _mm256_and_si256(y3, y10); + y4 = _mm256_and_si256(y4, y10); + y5 = _mm256_and_si256(y5, y10); + y6 = _mm256_and_si256(y6, y10); + y7 = _mm256_and_si256(y7, y10); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 736), y7); + y0 = _mm256_set1_epi32((int)WC_L32(rdi, 96)); + y1 = _mm256_set1_epi32((int)WC_L32(rdi, 100)); + y2 = _mm256_set1_epi32((int)WC_L32(rdi, 104)); + y3 = _mm256_set1_epi32((int)WC_L32(rdi, 108)); + y4 = _mm256_set1_epi32((int)WC_L32(rdi, 112)); + y5 = _mm256_set1_epi32((int)WC_L32(rdi, 116)); + y6 = _mm256_set1_epi32((int)WC_L32(rdi, 120)); + y7 = _mm256_set1_epi32((int)WC_L32(rdi, 124)); + y0 = _mm256_srlv_epi32(y0, y9); + y1 = _mm256_srlv_epi32(y1, y9); + y2 = _mm256_srlv_epi32(y2, y9); + y3 = _mm256_srlv_epi32(y3, y9); + y4 = _mm256_srlv_epi32(y4, y9); + y5 = _mm256_srlv_epi32(y5, y9); + y6 = _mm256_srlv_epi32(y6, y9); + y7 = _mm256_srlv_epi32(y7, y9); + y0 = _mm256_and_si256(y0, y10); + y1 = _mm256_and_si256(y1, y10); + y2 = _mm256_and_si256(y2, y10); + y3 = _mm256_and_si256(y3, y10); + y4 = _mm256_and_si256(y4, y10); + y5 = _mm256_and_si256(y5, y10); + y6 = _mm256_and_si256(y6, y10); + y7 = _mm256_and_si256(y7, y10); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y4 = _mm256_sub_epi32(y8, y4); + y5 = _mm256_sub_epi32(y8, y5); + y6 = _mm256_sub_epi32(y8, y6); + y7 = _mm256_sub_epi32(y8, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 864), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 992), y7); +} + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_avx2_shuff_0_even[] + WC_X64I_UNUSED = { + 0x00, 0x09, 0x0a, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x00, 0x09, 0x0a, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_avx2_shuff_0_odd[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x04, 0x05, 0x0e, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_avx2_shuff_1_even[] + WC_X64I_UNUSED = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x09, + 0x0a, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x00, 0x09, 0x0a, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_avx2_shuff_1_odd[] + WC_X64I_UNUSED = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x04, 0x05, + 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x04, 0x05, 0x0e, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_w1_88_avx2_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000006, 0x0000000c, 0x00000012, + 0x00000000, 0x00000006, 0x0000000c, 0x00000012, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_encode_w1_88_avx2(const sword32* w1, byte* w1e) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8; + + rdi = (word64)(size_t)w1; + rsi = (word64)(size_t)w1e; + + y4 = _mm256_setzero_si256(); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_88_avx2_shuff_0_even, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_88_avx2_shuff_0_odd, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_88_avx2_shuff_1_even, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_88_avx2_shuff_1_odd, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_encode_w1_88_avx2_vs, + 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_shuffle_epi8(y0, y5); + y0 = _mm256_shuffle_epi8(y0, y4); + y3 = _mm256_shuffle_epi8(y1, y7); + y1 = _mm256_shuffle_epi8(y1, y6); + y0 = _mm256_or_si256(y0, y2); + y0 = _mm256_or_si256(y0, y3); + y0 = _mm256_or_si256(y0, y1); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_or_si256(y0, y2); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rsi, 8) = (word32)_mm_extract_epi32(_mm256_castsi256_si128(y0), 2); + rsi = (word64)(rsi + 0xc); +} + +XALIGNED(16) static const word32 L_mldsa_encode_w1_32_avx2_vs_4[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000000, 0x00000004, 0x00000000, 0x00000004, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_encode_w1_32_avx2(const sword32* w1, byte* w1e) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8; + + rdi = (word64)(size_t)w1; + rsi = (word64)(size_t)w1e; + + y8 = _mm256_setzero_si256(); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_32_avx2_vs_4, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y4 = _mm256_sllv_epi32(y4, y8); + y5 = _mm256_sllv_epi32(y5, y8); + y6 = _mm256_sllv_epi32(y6, y8); + y7 = _mm256_sllv_epi32(y7, y8); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y4 = _mm256_sllv_epi32(y4, y8); + y5 = _mm256_sllv_epi32(y5, y8); + y6 = _mm256_sllv_epi32(y6, y8); + y7 = _mm256_sllv_epi32(y7, y8); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y4 = _mm256_sllv_epi32(y4, y8); + y5 = _mm256_sllv_epi32(y5, y8); + y6 = _mm256_sllv_epi32(y6, y8); + y7 = _mm256_sllv_epi32(y7, y8); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y0 = _mm256_sllv_epi32(y0, y8); + y1 = _mm256_sllv_epi32(y1, y8); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y4 = _mm256_sllv_epi32(y4, y8); + y5 = _mm256_sllv_epi32(y5, y8); + y6 = _mm256_sllv_epi32(y6, y8); + y7 = _mm256_sllv_epi32(y7, y8); + y0 = _mm256_packus_epi32(y0, y1); + y1 = _mm256_packus_epi32(y2, y3); + y2 = _mm256_packus_epi32(y4, y5); + y3 = _mm256_packus_epi32(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + y3 = _mm256_permute4x64_epi64(y3, 0xd8); + y0 = _mm256_hadd_epi16(y0, y1); + y1 = _mm256_hadd_epi16(y2, y3); + y2 = _mm256_hadd_epi16(y4, y5); + y3 = _mm256_hadd_epi16(y6, y7); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y1 = _mm256_permute4x64_epi64(y1, 0xd8); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y0); +} + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_avx2_d_max_half_m1[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_avx2_d_max_half[] + WC_X64I_UNUSED = { + 0x00001000, 0x00001000, 0x00001000, 0x00001000, + 0x00001000, 0x00001000, 0x00001000, 0x00001000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_avx2_vs_13[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000d, 0x00000002, 0x0000000f, + 0x00000004, 0x00000011, 0x00000006, 0x00000013, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_avx2_shuff_13_even[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0xff, 0x08, 0x09, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, + 0x02, 0x08, 0x09, 0x0a, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_avx2_shuff_13_odd[] + WC_X64I_UNUSED = { + 0xff, 0x05, 0x06, 0x07, 0x0d, 0x0e, 0x0f, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x06, 0x07, 0xff, 0x0e, 0x0f, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_avx2_vs_10[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000a, 0x00000004, 0x0000000e, + 0x00000000, 0x0000000a, 0x00000004, 0x0000000e, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_avx2_shuff_10_even[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x08, 0x09, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x08, + 0x09, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_avx2_shuff_10_odd[] + WC_X64I_UNUSED = { + 0xff, 0x05, 0x06, 0x0d, 0x0e, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x05, 0x06, + 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_vec_encode_t0_t1_avx2(const sword32* t, byte k, + byte* t0, byte* t1) +{ + word64 rdi, rsi, rdx, rcx; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), y6, y7, + y8, y9, y10, y11, y12, y13; + + rdi = (word64)(size_t)t; + rsi = (word64)(byte)k; + rdx = (word64)(size_t)t0; + rcx = (word64)(size_t)t1; + + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_avx2_d_max_half_m1, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_avx2_d_max_half, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_avx2_vs_13, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_avx2_shuff_13_even, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_avx2_shuff_13_odd, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_avx2_vs_10, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_avx2_shuff_10_even, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_avx2_shuff_10_odd, 0)); +L_mldsa_encode_t0_t1_avx2_loop: + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y4 = _mm256_add_epi32(y0, y6); + y5 = _mm256_add_epi32(y1, y6); + y4 = _mm256_srli_epi32(y4, 13); + y5 = _mm256_srli_epi32(y5, 13); + y2 = _mm256_slli_epi32(y4, 13); + y3 = _mm256_slli_epi32(y5, 13); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y7, y2); + y3 = _mm256_sub_epi32(y7, y3); + y2 = _mm256_sllv_epi32(y2, y8); + y3 = _mm256_sllv_epi32(y3, y8); + y0 = _mm256_shuffle_epi8(y2, y10); + y1 = _mm256_shuffle_epi8(y3, y10); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y2)); + rdx = (word64)(rdx + 0xd); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm256_castsi256_si128(y3)); + rdx = (word64)(rdx + 0xd); + y4 = _mm256_sllv_epi32(y4, y11); + y5 = _mm256_sllv_epi32(y5, y11); + y0 = _mm256_shuffle_epi8(y4, y13); + y1 = _mm256_shuffle_epi8(y5, y13); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y4 = _mm256_or_si256(y4, y0); + y5 = _mm256_or_si256(y5, y1); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y4)); + rcx = (word64)(rcx + 0xa); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm256_castsi256_si128(y5)); + rcx = (word64)(rcx + 0xa); + rdi = (word64)(rdi + 0x400); + rsi = (rsi & ~(word64)0xff) | ((word64)(byte)((byte)rsi - 1) & 0xff); + if (((byte)rsi) != (0)) { + goto L_mldsa_encode_t0_t1_avx2_loop; + } +} + +XALIGNED(16) static const byte L_mldsa_decode_t0_avx2_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0x03, + 0x03, 0x04, 0x05, 0xff, 0x04, 0x05, 0x06, 0x07, + 0x06, 0x07, 0x08, 0xff, 0xff, 0x08, 0x09, 0xff, + 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0x0b, 0x0c, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_avx2_shuff_1[] + WC_X64I_UNUSED = { + 0x05, 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, 0x08, + 0x08, 0x09, 0xff, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x03, 0x04, 0x05, 0xff, 0xff, 0x05, 0x06, 0xff, + 0x06, 0x07, 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_avx2_shuff_2[] + WC_X64I_UNUSED = { + 0x02, 0x03, 0xff, 0xff, 0xff, 0x03, 0x04, 0x05, + 0x05, 0x06, 0xff, 0xff, 0x06, 0x07, 0x08, 0xff, + 0x00, 0x01, 0x02, 0xff, 0xff, 0x02, 0x03, 0xff, + 0x03, 0x04, 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_avx2_shuff_3[] + WC_X64I_UNUSED = { + 0x07, 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, + 0x0a, 0x0b, 0xff, 0xff, 0x0b, 0x0c, 0x0d, 0xff, + 0x05, 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0xff, + 0x08, 0x09, 0x0a, 0xff, 0xff, 0xff, 0x0a, 0x0b, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_avx2_shuff_4[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0x07, + 0x07, 0x08, 0xff, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0x02, 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0xff, + 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_avx2_shuff_5[] + WC_X64I_UNUSED = { + 0x01, 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0x04, + 0x04, 0x05, 0xff, 0xff, 0x05, 0x06, 0x07, 0xff, + 0x07, 0x08, 0x09, 0xff, 0xff, 0x09, 0x0a, 0xff, + 0x0a, 0x0b, 0x0c, 0xff, 0xff, 0xff, 0x0c, 0x0d, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_avx2_shuff_6[] + WC_X64I_UNUSED = { + 0x06, 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, 0x09, + 0x09, 0x0a, 0xff, 0xff, 0x0a, 0x0b, 0x0c, 0xff, + 0x04, 0x05, 0x06, 0xff, 0xff, 0x06, 0x07, 0x08, + 0x07, 0x08, 0x09, 0xff, 0xff, 0xff, 0x09, 0x0a, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_avx2_shuff_7[] + WC_X64I_UNUSED = { + 0x03, 0x04, 0xff, 0xff, 0xff, 0x04, 0x05, 0x06, + 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0x09, 0xff, + 0x01, 0x02, 0x03, 0xff, 0xff, 0x03, 0x04, 0xff, + 0x04, 0x05, 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t0_avx2_vs_8[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000d, 0x00000002, 0x00000007, + 0x00000004, 0x00000009, 0x00000006, 0x00000013, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t0_avx2_mask[] + WC_X64I_UNUSED = { + 0x00001fff, 0x00001fff, 0x00001fff, 0x00001fff, + 0x00001fff, 0x00001fff, 0x00001fff, 0x00001fff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t0_avx2_d_max_half[] + WC_X64I_UNUSED = { + 0x00001000, 0x00001000, 0x00001000, 0x00001000, + 0x00001000, 0x00001000, 0x00001000, 0x00001000, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_decode_t0_avx2(const byte* t0, sword32* t) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)t0; + rsi = (word64)(size_t)t; + + y5 = _mm256_setzero_si256(); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_avx2_shuff_0, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_avx2_shuff_1, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_avx2_shuff_2, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_avx2_shuff_3, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_avx2_shuff_4, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_avx2_shuff_5, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_avx2_shuff_6, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_avx2_shuff_7, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_decode_t0_avx2_vs_8, + 0)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_decode_t0_avx2_mask, + 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_avx2_d_max_half, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + /* 1/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x44); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y4); + /* 2/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y4); + /* 3/32 */ + y0 = _mm256_permute2x128_si256(y0, y1, 0x21); + y4 = _mm256_permute4x64_epi64(y0, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y4); + /* 4/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0x94); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y4); + /* 5/32 */ + y1 = _mm256_permute2x128_si256(y1, y2, 0x21); + y4 = _mm256_permute4x64_epi64(y1, 0x94); + y4 = _mm256_shuffle_epi8(y4, y9); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y4); + /* 6/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0x44); + y4 = _mm256_shuffle_epi8(y4, y10); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y4); + /* 7/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y11); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y4); + /* 8/32 */ + y2 = _mm256_permute2x128_si256(y2, y3, 0x21); + y4 = _mm256_permute4x64_epi64(y2, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y12); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y4); + /* 9/32 */ + y4 = _mm256_permute4x64_epi64(y3, 0x99); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y4); + /* 10/32 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y3 = _mm256_permute2x128_si256(y3, y0, 0x21); + y4 = _mm256_permute4x64_epi64(y3, 0x94); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y4); + /* 11/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x94); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y4); + /* 12/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y4); + /* 13/32 */ + y0 = _mm256_permute2x128_si256(y0, y1, 0x21); + y4 = _mm256_permute4x64_epi64(y0, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y9); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y4); + /* 14/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0x99); + y4 = _mm256_shuffle_epi8(y4, y10); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y4); + /* 15/32 */ + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y1 = _mm256_permute2x128_si256(y1, y2, 0x21); + y4 = _mm256_permute4x64_epi64(y1, 0x94); + y4 = _mm256_shuffle_epi8(y4, y11); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y4); + /* 16/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0x94); + y4 = _mm256_shuffle_epi8(y4, y12); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y4); + /* 17/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0xee); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 512), y4); + /* 18/32 */ + y2 = _mm256_permute2x128_si256(y2, y3, 0x21); + y4 = _mm256_permute4x64_epi64(y2, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 544), y4); + /* 19/32 */ + y4 = _mm256_permute4x64_epi64(y3, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 576), y4); + /* 20/32 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y3 = _mm256_permute2x128_si256(y3, y0, 0x21); + y4 = _mm256_permute4x64_epi64(y3, 0x94); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 608), y4); + /* 21/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x94); + y4 = _mm256_shuffle_epi8(y4, y9); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 640), y4); + /* 22/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0xee); + y4 = _mm256_shuffle_epi8(y4, y10); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 672), y4); + /* 23/32 */ + y0 = _mm256_permute2x128_si256(y0, y1, 0x21); + y4 = _mm256_permute4x64_epi64(y0, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y11); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 704), y4); + /* 24/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y12); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 736), y4); + /* 25/32 */ + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y1 = _mm256_permute2x128_si256(y1, y2, 0x21); + y4 = _mm256_permute4x64_epi64(y1, 0x99); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 768), y4); + /* 26/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0x94); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 800), y4); + /* 27/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0x3e); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 832), y4); + /* 28/32 */ + y2 = _mm256_permute2x128_si256(y2, y3, 0x21); + y4 = _mm256_permute4x64_epi64(y2, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 864), y4); + /* 29/32 */ + y4 = _mm256_permute4x64_epi64(y3, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y9); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 896), y4); + /* 30/32 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y3 = _mm256_permute2x128_si256(y3, y0, 0x21); + y4 = _mm256_permute4x64_epi64(y3, 0x99); + y4 = _mm256_shuffle_epi8(y4, y10); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 928), y4); + /* 31/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x94); + y4 = _mm256_shuffle_epi8(y4, y11); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 960), y4); + /* 32/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x3e); + y4 = _mm256_shuffle_epi8(y4, y12); + y4 = _mm256_srlv_epi32(y4, y13); + y4 = _mm256_and_si256(y4, y14); + y4 = _mm256_sub_epi32(y15, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 992), y4); +} + +XALIGNED(16) static const byte L_mldsa_decode_t1_avx2_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0xff, + 0x02, 0x03, 0xff, 0xff, 0xff, 0x03, 0x04, 0xff, + 0x05, 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, 0xff, + 0x07, 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t1_avx2_shuff_1[] + WC_X64I_UNUSED = { + 0x02, 0x03, 0xff, 0xff, 0xff, 0x03, 0x04, 0xff, + 0x04, 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, + 0x07, 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, + 0x09, 0x0a, 0x08, 0xff, 0xff, 0x0a, 0x0b, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t1_avx2_shuff_2[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, + 0x06, 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, 0xff, + 0x01, 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0xff, + 0x03, 0x04, 0xff, 0xff, 0xff, 0x04, 0x05, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t1_avx2_shuff_3[] + WC_X64I_UNUSED = { + 0x06, 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, 0xff, + 0x08, 0x09, 0xff, 0xff, 0xff, 0x09, 0x0a, 0xff, + 0x03, 0x04, 0xff, 0xff, 0xff, 0x04, 0x05, 0xff, + 0x05, 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t1_avx2_vs_8[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000a, 0x00000004, 0x0000000e, + 0x00000000, 0x0000000a, 0x00000004, 0x0000000e, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t1_avx2_mask[] + WC_X64I_UNUSED = { + 0x000003ff, 0x000003ff, 0x000003ff, 0x000003ff, + 0x000003ff, 0x000003ff, 0x000003ff, 0x000003ff, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_decode_t1_avx2(const byte* t1, sword32* t) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10; + + rdi = (word64)(size_t)t1; + rsi = (word64)(size_t)t; + + y5 = _mm256_setzero_si256(); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_avx2_shuff_0, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_avx2_shuff_1, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_avx2_shuff_2, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_avx2_shuff_3, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_decode_t1_avx2_vs_8, + 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_decode_t1_avx2_mask, + 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + /* 1/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x44); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y4); + /* 2/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x99); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y4); + /* 3/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x3e); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y4); + /* 4/32 */ + y0 = _mm256_permute2x128_si256(y0, y1, 0x21); + y4 = _mm256_permute4x64_epi64(y0, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y4); + /* 5/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0x99); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y4); + /* 6/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0xee); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y4); + /* 7/32 */ + y1 = _mm256_permute2x128_si256(y1, y2, 0x21); + y4 = _mm256_permute4x64_epi64(y1, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y4); + /* 8/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0x94); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y4); + /* 9/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0xee); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y4); + /* 10/32 */ + y2 = _mm256_permute2x128_si256(y2, y3, 0x21); + y4 = _mm256_permute4x64_epi64(y2, 0x99); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y4); + /* 11/32 */ + y4 = _mm256_permute4x64_epi64(y3, 0x94); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y4); + /* 12/32 */ + y4 = _mm256_permute4x64_epi64(y3, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y4); + /* 13/32 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y3 = _mm256_permute2x128_si256(y3, y0, 0x21); + y4 = _mm256_permute4x64_epi64(y3, 0x99); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y4); + /* 14/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x44); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y4); + /* 15/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y4); + /* 16/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x3e); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y4); + /* 17/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0x44); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 512), y4); + /* 18/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0x99); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 544), y4); + /* 19/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0x3e); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 576), y4); + /* 20/32 */ + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y1 = _mm256_permute2x128_si256(y1, y2, 0x21); + y4 = _mm256_permute4x64_epi64(y1, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 608), y4); + /* 21/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0x99); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 640), y4); + /* 22/32 */ + y4 = _mm256_permute4x64_epi64(y2, 0xee); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 672), y4); + /* 23/32 */ + y2 = _mm256_permute2x128_si256(y2, y3, 0x21); + y4 = _mm256_permute4x64_epi64(y2, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 704), y4); + /* 24/32 */ + y4 = _mm256_permute4x64_epi64(y3, 0x94); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 736), y4); + /* 25/32 */ + y4 = _mm256_permute4x64_epi64(y3, 0xee); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 768), y4); + /* 26/32 */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y3 = _mm256_permute2x128_si256(y3, y0, 0x21); + y4 = _mm256_permute4x64_epi64(y3, 0x99); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 800), y4); + /* 27/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0x94); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 832), y4); + /* 28/32 */ + y4 = _mm256_permute4x64_epi64(y0, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 864), y4); + /* 29/32 */ + y0 = _mm256_permute2x128_si256(y0, y1, 0x21); + y4 = _mm256_permute4x64_epi64(y0, 0x99); + y4 = _mm256_shuffle_epi8(y4, y5); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 896), y4); + /* 30/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0x44); + y4 = _mm256_shuffle_epi8(y4, y6); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 928), y4); + /* 31/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0xe9); + y4 = _mm256_shuffle_epi8(y4, y7); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 960), y4); + /* 32/32 */ + y4 = _mm256_permute4x64_epi64(y1, 0x3e); + y4 = _mm256_shuffle_epi8(y4, y8); + y4 = _mm256_srlv_epi32(y4, y9); + y4 = _mm256_and_si256(y4, y10); + y4 = _mm256_slli_epi32(y4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 992), y4); +} + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_17_avx2_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff, + 0x04, 0x05, 0x06, 0xff, 0x06, 0x07, 0x08, 0xff, + 0xff, 0x01, 0x02, 0x03, 0xff, 0x03, 0x04, 0x05, + 0xff, 0x05, 0x06, 0x07, 0xff, 0x07, 0x08, 0x09, +}; + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_17_avx2_shuff_1[] + WC_X64I_UNUSED = { + 0x02, 0x03, 0x04, 0xff, 0x04, 0x05, 0x06, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0xff, 0x03, 0x04, 0x05, 0xff, 0x05, 0x06, 0x07, + 0xff, 0x07, 0x08, 0x09, 0xff, 0x09, 0x0a, 0x0b, +}; + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_17_avx2_shuff_2[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0x06, 0xff, 0x06, 0x07, 0x08, 0xff, + 0x08, 0x09, 0x0a, 0xff, 0x0a, 0x0b, 0x0c, 0xff, + 0xff, 0x05, 0x06, 0x07, 0xff, 0x07, 0x08, 0x09, + 0xff, 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0x0d, +}; + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_17_avx2_shuff_3[] + WC_X64I_UNUSED = { + 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0x0a, 0x0b, 0x0c, 0xff, 0x0c, 0x0d, 0x0e, 0xff, + 0xff, 0x07, 0x08, 0x09, 0xff, 0x09, 0x0a, 0x0b, + 0xff, 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_17_avx2_vs_8[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000002, 0x00000004, 0x00000006, + 0x00000008, 0x0000000a, 0x0000000c, 0x0000000e, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_17_avx2_mask[] + WC_X64I_UNUSED = { + 0x0003ffff, 0x0003ffff, 0x0003ffff, 0x0003ffff, + 0x0003ffff, 0x0003ffff, 0x0003ffff, 0x0003ffff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_17_avx2_gamma17[] + WC_X64I_UNUSED = { + 0x00020000, 0x00020000, 0x00020000, 0x00020000, + 0x00020000, 0x00020000, 0x00020000, 0x00020000, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_decode_gamma1_17_avx2(const byte* s, sword32* z) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)z; + + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_avx2_shuff_0, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_avx2_shuff_1, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_avx2_shuff_2, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_avx2_shuff_3, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_avx2_vs_8, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_avx2_mask, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_avx2_gamma17, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + /* 0/15 */ + y6 = _mm256_permute4x64_epi64(y0, 0x94); + y6 = _mm256_shuffle_epi8(y6, y7); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y6); + /* 1/15 */ + y6 = _mm256_permute2x128_si256(y0, y1, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0x94); + y6 = _mm256_shuffle_epi8(y6, y8); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y6); + /* 2/15 */ + y6 = _mm256_permute4x64_epi64(y1, 0x94); + y6 = _mm256_shuffle_epi8(y6, y9); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y6); + /* 3/15 */ + y6 = _mm256_permute2x128_si256(y1, y2, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0x94); + y6 = _mm256_shuffle_epi8(y6, y10); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y6); + /* 4/15 */ + y6 = _mm256_permute4x64_epi64(y2, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y7); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y6); + /* 5/15 */ + y6 = _mm256_permute2x128_si256(y2, y3, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y8); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y6); + /* 6/15 */ + y6 = _mm256_permute4x64_epi64(y3, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y9); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y6); + /* 7/15 */ + y6 = _mm256_permute2x128_si256(y3, y4, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y10); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y6); + /* 8/15 */ + y6 = _mm256_permute2x128_si256(y4, y5, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0x94); + y6 = _mm256_shuffle_epi8(y6, y7); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y6); + /* 9/15 */ + y6 = _mm256_permute4x64_epi64(y5, 0x94); + y6 = _mm256_shuffle_epi8(y6, y8); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y6); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + /* 10/15 */ + y6 = _mm256_permute2x128_si256(y5, y0, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0x94); + y6 = _mm256_shuffle_epi8(y6, y9); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y6); + /* 11/15 */ + y6 = _mm256_permute4x64_epi64(y0, 0x94); + y6 = _mm256_shuffle_epi8(y6, y10); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y6); + /* 12/15 */ + y6 = _mm256_permute2x128_si256(y0, y1, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y7); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y6); + /* 13/15 */ + y6 = _mm256_permute4x64_epi64(y1, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y8); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y6); + /* 14/15 */ + y6 = _mm256_permute2x128_si256(y1, y2, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y9); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y6); + /* 15/15 */ + y6 = _mm256_permute4x64_epi64(y2, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y10); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y6); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + /* 0/15 */ + y6 = _mm256_permute4x64_epi64(y0, 0x94); + y6 = _mm256_shuffle_epi8(y6, y7); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 512), y6); + /* 1/15 */ + y6 = _mm256_permute2x128_si256(y0, y1, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0x94); + y6 = _mm256_shuffle_epi8(y6, y8); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 544), y6); + /* 2/15 */ + y6 = _mm256_permute4x64_epi64(y1, 0x94); + y6 = _mm256_shuffle_epi8(y6, y9); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 576), y6); + /* 3/15 */ + y6 = _mm256_permute2x128_si256(y1, y2, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0x94); + y6 = _mm256_shuffle_epi8(y6, y10); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 608), y6); + /* 4/15 */ + y6 = _mm256_permute4x64_epi64(y2, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y7); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 640), y6); + /* 5/15 */ + y6 = _mm256_permute2x128_si256(y2, y3, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y8); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 672), y6); + /* 6/15 */ + y6 = _mm256_permute4x64_epi64(y3, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y9); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 704), y6); + /* 7/15 */ + y6 = _mm256_permute2x128_si256(y3, y4, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y10); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 736), y6); + /* 8/15 */ + y6 = _mm256_permute2x128_si256(y4, y5, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0x94); + y6 = _mm256_shuffle_epi8(y6, y7); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 768), y6); + /* 9/15 */ + y6 = _mm256_permute4x64_epi64(y5, 0x94); + y6 = _mm256_shuffle_epi8(y6, y8); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 800), y6); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + /* 10/15 */ + y6 = _mm256_permute2x128_si256(y5, y0, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0x94); + y6 = _mm256_shuffle_epi8(y6, y9); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 832), y6); + /* 11/15 */ + y6 = _mm256_permute4x64_epi64(y0, 0x94); + y6 = _mm256_shuffle_epi8(y6, y10); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 864), y6); + /* 12/15 */ + y6 = _mm256_permute2x128_si256(y0, y1, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y7); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 896), y6); + /* 13/15 */ + y6 = _mm256_permute4x64_epi64(y1, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y8); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 928), y6); + /* 14/15 */ + y6 = _mm256_permute2x128_si256(y1, y2, 0x21); + y6 = _mm256_permute4x64_epi64(y6, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y9); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 960), y6); + /* 15/15 */ + y6 = _mm256_permute4x64_epi64(y2, 0xe9); + y6 = _mm256_shuffle_epi8(y6, y10); + y6 = _mm256_srlv_epi32(y6, y11); + y6 = _mm256_and_si256(y6, y12); + y6 = _mm256_sub_epi32(y13, y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 992), y6); +} + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_20_avx2_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff, + 0x05, 0x06, 0x07, 0xff, 0x07, 0x08, 0x09, 0xff, + 0xff, 0x02, 0x03, 0x04, 0xff, 0x04, 0x05, 0x06, + 0xff, 0x07, 0x08, 0x09, 0xff, 0x09, 0x0a, 0x0b, +}; + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_20_avx2_shuff_1[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0x06, 0xff, 0x06, 0x07, 0x08, 0xff, + 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0x0d, 0xff, + 0xff, 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0x0a, + 0xff, 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_20_avx2_vs_8[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000008, 0x0000000c, 0x00000008, 0x0000000c, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_20_avx2_mask[] + WC_X64I_UNUSED = { + 0x000fffff, 0x000fffff, 0x000fffff, 0x000fffff, + 0x000fffff, 0x000fffff, 0x000fffff, 0x000fffff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_20_avx2_gamma19[] + WC_X64I_UNUSED = { + 0x00080000, 0x00080000, 0x00080000, 0x00080000, + 0x00080000, 0x00080000, 0x00080000, 0x00080000, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_decode_gamma1_19_avx2(const byte* s, sword32* z) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)z; + + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_20_avx2_shuff_0, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_20_avx2_shuff_1, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_20_avx2_vs_8, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_20_avx2_mask, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_20_avx2_gamma19, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + /* 0/7 */ + y5 = _mm256_permute4x64_epi64(y0, 0x94); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y5); + /* 1/7 */ + y5 = _mm256_permute2x128_si256(y0, y1, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0x94); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y5); + /* 2/7 */ + y5 = _mm256_permute4x64_epi64(y1, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y5); + /* 3/7 */ + y5 = _mm256_permute2x128_si256(y1, y2, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y5); + /* 4/7 */ + y5 = _mm256_permute2x128_si256(y2, y3, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0x94); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y5); + /* 5/7 */ + y5 = _mm256_permute4x64_epi64(y3, 0x94); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y5); + /* 6/7 */ + y5 = _mm256_permute2x128_si256(y3, y4, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y5); + /* 7/7 */ + y5 = _mm256_permute4x64_epi64(y4, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + /* 0/7 */ + y5 = _mm256_permute4x64_epi64(y0, 0x94); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y5); + /* 1/7 */ + y5 = _mm256_permute2x128_si256(y0, y1, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0x94); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y5); + /* 2/7 */ + y5 = _mm256_permute4x64_epi64(y1, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y5); + /* 3/7 */ + y5 = _mm256_permute2x128_si256(y1, y2, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y5); + /* 4/7 */ + y5 = _mm256_permute2x128_si256(y2, y3, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0x94); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y5); + /* 5/7 */ + y5 = _mm256_permute4x64_epi64(y3, 0x94); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y5); + /* 6/7 */ + y5 = _mm256_permute2x128_si256(y3, y4, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y5); + /* 7/7 */ + y5 = _mm256_permute4x64_epi64(y4, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + /* 0/7 */ + y5 = _mm256_permute4x64_epi64(y0, 0x94); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 512), y5); + /* 1/7 */ + y5 = _mm256_permute2x128_si256(y0, y1, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0x94); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 544), y5); + /* 2/7 */ + y5 = _mm256_permute4x64_epi64(y1, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 576), y5); + /* 3/7 */ + y5 = _mm256_permute2x128_si256(y1, y2, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 608), y5); + /* 4/7 */ + y5 = _mm256_permute2x128_si256(y2, y3, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0x94); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 640), y5); + /* 5/7 */ + y5 = _mm256_permute4x64_epi64(y3, 0x94); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 672), y5); + /* 6/7 */ + y5 = _mm256_permute2x128_si256(y3, y4, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 704), y5); + /* 7/7 */ + y5 = _mm256_permute4x64_epi64(y4, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 736), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + /* 0/7 */ + y5 = _mm256_permute4x64_epi64(y0, 0x94); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 768), y5); + /* 1/7 */ + y5 = _mm256_permute2x128_si256(y0, y1, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0x94); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 800), y5); + /* 2/7 */ + y5 = _mm256_permute4x64_epi64(y1, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 832), y5); + /* 3/7 */ + y5 = _mm256_permute2x128_si256(y1, y2, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 864), y5); + /* 4/7 */ + y5 = _mm256_permute2x128_si256(y2, y3, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0x94); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 896), y5); + /* 5/7 */ + y5 = _mm256_permute4x64_epi64(y3, 0x94); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 928), y5); + /* 6/7 */ + y5 = _mm256_permute2x128_si256(y3, y4, 0x21); + y5 = _mm256_permute4x64_epi64(y5, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y6); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 960), y5); + /* 7/7 */ + y5 = _mm256_permute4x64_epi64(y4, 0xe9); + y5 = _mm256_shuffle_epi8(y5, y7); + y5 = _mm256_srlv_epi32(y5, y8); + y5 = _mm256_and_si256(y5, y9); + y5 = _mm256_sub_epi32(y10, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 992), y5); +} + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_17_avx2_gamma17[] + WC_X64I_UNUSED = { + 0x00020000, 0x00020000, 0x00020000, 0x00020000, + 0x00020000, 0x00020000, 0x00020000, 0x00020000, +}; + +XALIGNED(16) static const byte L_mldsa_encode_gamma1_17_avx2_shuff_even[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_gamma1_17_avx2_shuff_odd[] + WC_X64I_UNUSED = { + 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0x0c, 0x0d, + 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0x0c, 0x0d, + 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_17_avx2_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000002, 0x00000004, 0x00000006, + 0x00000000, 0x00000002, 0x00000004, 0x00000006, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_encode_gamma1_17_avx2(const sword32* z, byte* s) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11; + + rdi = (word64)(size_t)z; + rsi = (word64)(size_t)s; + + y8 = _mm256_setzero_si256(); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_gamma1_17_avx2_gamma17, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_gamma1_17_avx2_shuff_even, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_gamma1_17_avx2_shuff_odd, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_gamma1_17_avx2_vs, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y0), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y4)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y4), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y1), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y5)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y5), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y2), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y6)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y6), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y3), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y7)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y7), 8); + rsi = (word64)(rsi + 0x12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y0), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y4)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y4), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y1), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y5)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y5), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y2), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y6)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y6), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y3), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y7)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y7), 8); + rsi = (word64)(rsi + 0x12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y0), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y4)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y4), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y1), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y5)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y5), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y2), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y6)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y6), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y3), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y7)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y7), 8); + rsi = (word64)(rsi + 0x12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y0), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y4)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y4), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y1), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y5)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y5), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y2), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y6)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y6), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y3), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y7)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y7), 8); + rsi = (word64)(rsi + 0x12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y0), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y4)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y4), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y1), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y5)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y5), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y2), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y6)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y6), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y3), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y7)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y7), 8); + rsi = (word64)(rsi + 0x12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y0), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y4)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y4), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y1), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y5)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y5), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y2), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y6)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y6), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y3), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y7)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y7), 8); + rsi = (word64)(rsi + 0x12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y0), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y4)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y4), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y1), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y5)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y5), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y2), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y6)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y6), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y3), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y7)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y7), 8); + rsi = (word64)(rsi + 0x12); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y0), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y4)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y4), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y1), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y5)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y5), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y2), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y6)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y6), 8); + rsi = (word64)(rsi + 0x12); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S8(rsi, 8) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y3), 8); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 9), _mm256_castsi256_si128(y7)); + WC_S8(rsi, 17) = (word8)_mm_extract_epi8(_mm256_castsi256_si128(y7), 8); + rsi = (word64)(rsi + 0x12); +} + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_19_avx2_gamma19[] + WC_X64I_UNUSED = { + 0x00080000, 0x00080000, 0x00080000, 0x00080000, + 0x00080000, 0x00080000, 0x00080000, 0x00080000, +}; + +XALIGNED(16) static const byte L_mldsa_encode_gamma1_19_avx2_shuff_even[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0xff, 0x08, 0x09, 0x0a, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0xff, 0xff, 0x08, 0x09, 0x0a, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_gamma1_19_avx2_shuff_odd[] + WC_X64I_UNUSED = { + 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0xff, 0x0c, + 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0xff, 0x0c, + 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_19_avx2_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000000, 0x00000004, 0x00000000, 0x00000004, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_encode_gamma1_19_avx2(const sword32* z, byte* s) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11; + + rdi = (word64)(size_t)z; + rsi = (word64)(size_t)s; + + y8 = _mm256_setzero_si256(); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_gamma1_19_avx2_gamma19, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_gamma1_19_avx2_shuff_even, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_gamma1_19_avx2_shuff_odd, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_gamma1_19_avx2_vs, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y0), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y4)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y4), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y1), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y5)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y5), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y2), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y6)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y6), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y3), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y7)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y7), 4); + rsi = (word64)(rsi + 0x14); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y0), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y4)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y4), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y1), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y5)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y5), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y2), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y6)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y6), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y3), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y7)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y7), 4); + rsi = (word64)(rsi + 0x14); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y0), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y4)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y4), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y1), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y5)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y5), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y2), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y6)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y6), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y3), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y7)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y7), 4); + rsi = (word64)(rsi + 0x14); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y0), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y4)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y4), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y1), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y5)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y5), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y2), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y6)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y6), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y3), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y7)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y7), 4); + rsi = (word64)(rsi + 0x14); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y0), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y4)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y4), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y1), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y5)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y5), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y2), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y6)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y6), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y3), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y7)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y7), 4); + rsi = (word64)(rsi + 0x14); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y0), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y4)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y4), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y1), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y5)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y5), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y2), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y6)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y6), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y3), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y7)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y7), 4); + rsi = (word64)(rsi + 0x14); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y0), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y4)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y4), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y1), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y5)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y5), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y2), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y6)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y6), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y3), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y7)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y7), 4); + rsi = (word64)(rsi + 0x14); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y0 = _mm256_sub_epi32(y8, y0); + y1 = _mm256_sub_epi32(y8, y1); + y2 = _mm256_sub_epi32(y8, y2); + y3 = _mm256_sub_epi32(y8, y3); + y0 = _mm256_sllv_epi32(y0, y11); + y1 = _mm256_sllv_epi32(y1, y11); + y2 = _mm256_sllv_epi32(y2, y11); + y3 = _mm256_sllv_epi32(y3, y11); + y4 = _mm256_shuffle_epi8(y0, y10); + y5 = _mm256_shuffle_epi8(y1, y10); + y6 = _mm256_shuffle_epi8(y2, y10); + y7 = _mm256_shuffle_epi8(y3, y10); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_shuffle_epi8(y2, y9); + y3 = _mm256_shuffle_epi8(y3, y9); + y0 = _mm256_or_si256(y0, y4); + y1 = _mm256_or_si256(y1, y5); + y2 = _mm256_or_si256(y2, y6); + y3 = _mm256_or_si256(y3, y7); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y5 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y2, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y0)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y0), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y4)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y4), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y1)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y1), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y5)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y5), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y2)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y2), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y6)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y6), 4); + rsi = (word64)(rsi + 0x14); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 0), _mm256_castsi256_si128(y3)); + WC_S16(rsi, 8) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y3), 4); + _mm_storel_epi64((__m128i*)WC_PW(rsi, 10), _mm256_castsi256_si128(y7)); + WC_S16(rsi, 18) = (word16)_mm_extract_epi16(_mm256_castsi256_si128(y7), 4); + rsi = (word64)(rsi + 0x14); +} + +XALIGNED(16) static const word32 L_mldsa_decompose_q88_avx2_q_low_88[] + WC_X64I_UNUSED = { + 0x00017400, 0x00017400, 0x00017400, 0x00017400, + 0x00017400, 0x00017400, 0x00017400, 0x00017400, +}; + +XALIGNED(16) static const word32 L_mldsa_decompose_q88_avx2_q_low_88_2[] + WC_X64I_UNUSED = { + 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800, + 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800, +}; + +XALIGNED(16) static const word32 L_mldsa_decompose_q88_avx2_q_2[] + WC_X64I_UNUSED = { + 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4, + 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4, +}; + +XALIGNED(16) static const word32 L_mldsa_decompose_q88_avx2_44[] + WC_X64I_UNUSED = { + 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c, + 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_decompose_q88_avx2(const sword32* r, sword32* r0, + sword32* r1) +{ + word64 rdi, rsi, rdx; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decompose_q88_avx2_q_low_88, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decompose_q88_avx2_q_low_88_2, 0)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decompose_q88_avx2_q_2, 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decompose_q88_avx2_44, 0)); + /* 1/4 vectors */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 544), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 576), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 608), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 512), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 544), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 576), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 608), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 736), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 640), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 672), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 704), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 736), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 768), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 800), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 832), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 864), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 768), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 800), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 832), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 864), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 992), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 896), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 928), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 960), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 992), y11); + /* 2/4 vectors */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1024)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1056)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1088)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1120)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1024), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1056), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1088), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1120), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1024), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1056), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1088), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1120), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1152)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1184)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1216)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1248)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1152), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1184), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1216), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1248), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1152), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1184), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1216), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1248), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1280)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1312)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1344)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1376)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1280), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1312), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1344), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1376), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1280), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1312), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1344), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1376), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1408)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1440)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1472)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1504)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1408), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1440), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1472), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1504), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1408), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1440), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1472), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1504), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1536)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1568)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1600)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1632)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1536), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1568), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1600), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1632), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1536), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1568), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1600), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1632), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1664)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1696)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1728)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1760)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1664), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1696), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1728), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1760), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1664), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1696), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1728), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1760), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1792)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1824)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1856)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1888)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1792), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1824), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1856), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1888), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1792), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1824), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1856), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1888), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1920)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1952)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1984)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2016)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1920), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1952), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 1984), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2016), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1920), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1952), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 1984), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2016), y11); + /* 3/4 vectors */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2048)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2080)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2112)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2144)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2048), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2080), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2112), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2144), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2048), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2080), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2112), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2144), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2176)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2208)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2240)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2272)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2176), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2208), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2240), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2272), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2176), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2208), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2240), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2272), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2304)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2336)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2368)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2400)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2304), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2336), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2368), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2400), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2304), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2336), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2368), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2400), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2432)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2464)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2496)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2528)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2432), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2464), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2496), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2528), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2432), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2464), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2496), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2528), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2560)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2592)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2624)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2656)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2560), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2592), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2624), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2656), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2560), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2592), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2624), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2656), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2688)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2720)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2752)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2784)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2688), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2720), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2752), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2784), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2688), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2720), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2752), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2784), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2816)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2848)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2880)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2912)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2816), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2848), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2880), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2912), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2816), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2848), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2880), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2912), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2944)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2976)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3008)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3040)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2944), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 2976), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3008), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3040), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2944), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 2976), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3008), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3040), y11); + /* 4/4 vectors */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3072)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3104)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3136)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3168)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3072), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3104), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3136), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3168), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3072), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3104), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3136), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3168), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3200)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3232)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3264)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3296)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3200), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3232), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3264), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3296), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3200), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3232), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3264), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3296), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3328)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3360)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3392)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3424)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3328), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3360), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3392), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3424), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3328), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3360), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3392), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3424), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3456)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3488)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3520)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3552)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3456), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3488), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3520), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3552), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3456), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3488), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3520), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3552), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3584)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3616)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3648)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3680)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3584), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3616), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3648), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3680), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3584), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3616), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3648), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3680), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3712)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3744)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3776)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3808)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3712), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3744), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3776), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3808), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3712), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3744), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3776), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3808), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3840)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3872)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3904)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3936)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3840), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3872), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3904), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3936), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3840), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3872), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3904), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3936), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3968)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4000)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4032)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4064)); + y8 = _mm256_mullo_epi32(y0, y15); + y9 = _mm256_mullo_epi32(y1, y15); + y10 = _mm256_mullo_epi32(y2, y15); + y11 = _mm256_mullo_epi32(y3, y15); + y8 = _mm256_add_epi32(y8, y14); + y9 = _mm256_add_epi32(y9, y14); + y10 = _mm256_add_epi32(y10, y14); + y11 = _mm256_add_epi32(y11, y14); + y8 = _mm256_srli_epi32(y8, 23); + y9 = _mm256_srli_epi32(y9, 23); + y10 = _mm256_srli_epi32(y10, 23); + y11 = _mm256_srli_epi32(y11, 23); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_cmpeq_epi32(y8, y15); + y1 = _mm256_cmpeq_epi32(y9, y15); + y2 = _mm256_cmpeq_epi32(y10, y15); + y3 = _mm256_cmpeq_epi32(y11, y15); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + y6 = _mm256_add_epi32(y6, y2); + y7 = _mm256_add_epi32(y7, y3); + y0 = _mm256_cmpgt_epi32(y15, y8); + y1 = _mm256_cmpgt_epi32(y15, y9); + y2 = _mm256_cmpgt_epi32(y15, y10); + y3 = _mm256_cmpgt_epi32(y15, y11); + y8 = _mm256_and_si256(y8, y0); + y9 = _mm256_and_si256(y9, y1); + y10 = _mm256_and_si256(y10, y2); + y11 = _mm256_and_si256(y11, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 3968), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 4000), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 4032), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 4064), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 3968), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 4000), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 4032), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 4064), y11); +} + +XALIGNED(16) static const word32 L_mldsa_decompose_q32_avx2_q_low_32[] + WC_X64I_UNUSED = { + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, +}; + +XALIGNED(16) static const word32 L_mldsa_decompose_q32_avx2_q_low_32_2[] + WC_X64I_UNUSED = { + 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00, + 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00, +}; + +XALIGNED(16) static const word32 L_mldsa_decompose_q32_avx2_q_low_32_m1[] + WC_X64I_UNUSED = { + 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff, + 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff, +}; + +XALIGNED(16) static const word32 L_mldsa_decompose_q32_avx2_mask[] + WC_X64I_UNUSED = { + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_decompose_q32_avx2(const sword32* r, byte k, + sword32* r0, sword32* r1) +{ + word64 rdi, rsi, rdx, rcx; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), y12, + y13, y14, y15; + + rdi = (word64)(size_t)r; + rsi = (word64)(byte)k; + rdx = (word64)(size_t)r0; + rcx = (word64)(size_t)r1; + + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decompose_q32_avx2_q_low_32, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decompose_q32_avx2_q_low_32_2, 0)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decompose_q32_avx2_q_low_32_m1, 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decompose_q32_avx2_mask, 0)); +L_mldsa_decompose_q32_avx2_start_256: + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y8 = _mm256_add_epi32(y0, y14); + y9 = _mm256_add_epi32(y1, y14); + y10 = _mm256_add_epi32(y2, y14); + y11 = _mm256_add_epi32(y3, y14); + y8 = _mm256_srli_epi32(y8, 19); + y9 = _mm256_srli_epi32(y9, 19); + y10 = _mm256_srli_epi32(y10, 19); + y11 = _mm256_srli_epi32(y11, 19); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi32(y8, 4); + y1 = _mm256_srli_epi32(y9, 4); + y2 = _mm256_srli_epi32(y10, 4); + y3 = _mm256_srli_epi32(y11, 4); + y4 = _mm256_sub_epi32(y4, y0); + y5 = _mm256_sub_epi32(y5, y1); + y6 = _mm256_sub_epi32(y6, y2); + y7 = _mm256_sub_epi32(y7, y3); + y8 = _mm256_and_si256(y8, y15); + y9 = _mm256_and_si256(y9, y15); + y10 = _mm256_and_si256(y10, y15); + y11 = _mm256_and_si256(y11, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_add_epi32(y0, y14); + y9 = _mm256_add_epi32(y1, y14); + y10 = _mm256_add_epi32(y2, y14); + y11 = _mm256_add_epi32(y3, y14); + y8 = _mm256_srli_epi32(y8, 19); + y9 = _mm256_srli_epi32(y9, 19); + y10 = _mm256_srli_epi32(y10, 19); + y11 = _mm256_srli_epi32(y11, 19); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi32(y8, 4); + y1 = _mm256_srli_epi32(y9, 4); + y2 = _mm256_srli_epi32(y10, 4); + y3 = _mm256_srli_epi32(y11, 4); + y4 = _mm256_sub_epi32(y4, y0); + y5 = _mm256_sub_epi32(y5, y1); + y6 = _mm256_sub_epi32(y6, y2); + y7 = _mm256_sub_epi32(y7, y3); + y8 = _mm256_and_si256(y8, y15); + y9 = _mm256_and_si256(y9, y15); + y10 = _mm256_and_si256(y10, y15); + y11 = _mm256_and_si256(y11, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 160), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 192), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 224), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y8 = _mm256_add_epi32(y0, y14); + y9 = _mm256_add_epi32(y1, y14); + y10 = _mm256_add_epi32(y2, y14); + y11 = _mm256_add_epi32(y3, y14); + y8 = _mm256_srli_epi32(y8, 19); + y9 = _mm256_srli_epi32(y9, 19); + y10 = _mm256_srli_epi32(y10, 19); + y11 = _mm256_srli_epi32(y11, 19); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi32(y8, 4); + y1 = _mm256_srli_epi32(y9, 4); + y2 = _mm256_srli_epi32(y10, 4); + y3 = _mm256_srli_epi32(y11, 4); + y4 = _mm256_sub_epi32(y4, y0); + y5 = _mm256_sub_epi32(y5, y1); + y6 = _mm256_sub_epi32(y6, y2); + y7 = _mm256_sub_epi32(y7, y3); + y8 = _mm256_and_si256(y8, y15); + y9 = _mm256_and_si256(y9, y15); + y10 = _mm256_and_si256(y10, y15); + y11 = _mm256_and_si256(y11, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 256), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 288), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 320), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 352), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y8 = _mm256_add_epi32(y0, y14); + y9 = _mm256_add_epi32(y1, y14); + y10 = _mm256_add_epi32(y2, y14); + y11 = _mm256_add_epi32(y3, y14); + y8 = _mm256_srli_epi32(y8, 19); + y9 = _mm256_srli_epi32(y9, 19); + y10 = _mm256_srli_epi32(y10, 19); + y11 = _mm256_srli_epi32(y11, 19); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi32(y8, 4); + y1 = _mm256_srli_epi32(y9, 4); + y2 = _mm256_srli_epi32(y10, 4); + y3 = _mm256_srli_epi32(y11, 4); + y4 = _mm256_sub_epi32(y4, y0); + y5 = _mm256_sub_epi32(y5, y1); + y6 = _mm256_sub_epi32(y6, y2); + y7 = _mm256_sub_epi32(y7, y3); + y8 = _mm256_and_si256(y8, y15); + y9 = _mm256_and_si256(y9, y15); + y10 = _mm256_and_si256(y10, y15); + y11 = _mm256_and_si256(y11, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 384), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 416), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 448), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 480), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y8 = _mm256_add_epi32(y0, y14); + y9 = _mm256_add_epi32(y1, y14); + y10 = _mm256_add_epi32(y2, y14); + y11 = _mm256_add_epi32(y3, y14); + y8 = _mm256_srli_epi32(y8, 19); + y9 = _mm256_srli_epi32(y9, 19); + y10 = _mm256_srli_epi32(y10, 19); + y11 = _mm256_srli_epi32(y11, 19); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi32(y8, 4); + y1 = _mm256_srli_epi32(y9, 4); + y2 = _mm256_srli_epi32(y10, 4); + y3 = _mm256_srli_epi32(y11, 4); + y4 = _mm256_sub_epi32(y4, y0); + y5 = _mm256_sub_epi32(y5, y1); + y6 = _mm256_sub_epi32(y6, y2); + y7 = _mm256_sub_epi32(y7, y3); + y8 = _mm256_and_si256(y8, y15); + y9 = _mm256_and_si256(y9, y15); + y10 = _mm256_and_si256(y10, y15); + y11 = _mm256_and_si256(y11, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 544), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 576), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 608), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 512), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 544), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 576), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 608), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y8 = _mm256_add_epi32(y0, y14); + y9 = _mm256_add_epi32(y1, y14); + y10 = _mm256_add_epi32(y2, y14); + y11 = _mm256_add_epi32(y3, y14); + y8 = _mm256_srli_epi32(y8, 19); + y9 = _mm256_srli_epi32(y9, 19); + y10 = _mm256_srli_epi32(y10, 19); + y11 = _mm256_srli_epi32(y11, 19); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi32(y8, 4); + y1 = _mm256_srli_epi32(y9, 4); + y2 = _mm256_srli_epi32(y10, 4); + y3 = _mm256_srli_epi32(y11, 4); + y4 = _mm256_sub_epi32(y4, y0); + y5 = _mm256_sub_epi32(y5, y1); + y6 = _mm256_sub_epi32(y6, y2); + y7 = _mm256_sub_epi32(y7, y3); + y8 = _mm256_and_si256(y8, y15); + y9 = _mm256_and_si256(y9, y15); + y10 = _mm256_and_si256(y10, y15); + y11 = _mm256_and_si256(y11, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 736), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 640), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 672), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 704), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 736), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y8 = _mm256_add_epi32(y0, y14); + y9 = _mm256_add_epi32(y1, y14); + y10 = _mm256_add_epi32(y2, y14); + y11 = _mm256_add_epi32(y3, y14); + y8 = _mm256_srli_epi32(y8, 19); + y9 = _mm256_srli_epi32(y9, 19); + y10 = _mm256_srli_epi32(y10, 19); + y11 = _mm256_srli_epi32(y11, 19); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi32(y8, 4); + y1 = _mm256_srli_epi32(y9, 4); + y2 = _mm256_srli_epi32(y10, 4); + y3 = _mm256_srli_epi32(y11, 4); + y4 = _mm256_sub_epi32(y4, y0); + y5 = _mm256_sub_epi32(y5, y1); + y6 = _mm256_sub_epi32(y6, y2); + y7 = _mm256_sub_epi32(y7, y3); + y8 = _mm256_and_si256(y8, y15); + y9 = _mm256_and_si256(y9, y15); + y10 = _mm256_and_si256(y10, y15); + y11 = _mm256_and_si256(y11, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 768), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 800), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 832), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 864), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 768), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 800), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 832), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 864), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_add_epi32(y0, y14); + y9 = _mm256_add_epi32(y1, y14); + y10 = _mm256_add_epi32(y2, y14); + y11 = _mm256_add_epi32(y3, y14); + y8 = _mm256_srli_epi32(y8, 19); + y9 = _mm256_srli_epi32(y9, 19); + y10 = _mm256_srli_epi32(y10, 19); + y11 = _mm256_srli_epi32(y11, 19); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y4 = _mm256_sub_epi32(y12, y4); + y5 = _mm256_sub_epi32(y12, y5); + y6 = _mm256_sub_epi32(y12, y6); + y7 = _mm256_sub_epi32(y12, y7); + y4 = _mm256_srli_epi32(y4, 31); + y5 = _mm256_srli_epi32(y5, 31); + y6 = _mm256_srli_epi32(y6, 31); + y7 = _mm256_srli_epi32(y7, 31); + y8 = _mm256_add_epi32(y8, y4); + y9 = _mm256_add_epi32(y9, y5); + y10 = _mm256_add_epi32(y10, y6); + y11 = _mm256_add_epi32(y11, y7); + y4 = _mm256_mullo_epi32(y8, y13); + y5 = _mm256_mullo_epi32(y9, y13); + y6 = _mm256_mullo_epi32(y10, y13); + y7 = _mm256_mullo_epi32(y11, y13); + y4 = _mm256_sub_epi32(y0, y4); + y5 = _mm256_sub_epi32(y1, y5); + y6 = _mm256_sub_epi32(y2, y6); + y7 = _mm256_sub_epi32(y3, y7); + y0 = _mm256_srli_epi32(y8, 4); + y1 = _mm256_srli_epi32(y9, 4); + y2 = _mm256_srli_epi32(y10, 4); + y3 = _mm256_srli_epi32(y11, 4); + y4 = _mm256_sub_epi32(y4, y0); + y5 = _mm256_sub_epi32(y5, y1); + y6 = _mm256_sub_epi32(y6, y2); + y7 = _mm256_sub_epi32(y7, y3); + y8 = _mm256_and_si256(y8, y15); + y9 = _mm256_and_si256(y9, y15); + y10 = _mm256_and_si256(y10, y15); + y11 = _mm256_and_si256(y11, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 992), y7); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 896), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 928), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 960), y10); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 992), y11); + rdi = (word64)(rdi + 0x400); + rdx = (word64)(rdx + 0x400); + rcx = (word64)(rcx + 0x400); + rsi = (word64)(rsi - 1); + if ((rsi) != (0)) { + goto L_mldsa_decompose_q32_avx2_start_256; + } +} + +XALIGNED(16) static const word32 L_mldsa_use_hint_88_avx2_q[] WC_X64I_UNUSED = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_88_avx2_q_low_88[] + WC_X64I_UNUSED = { + 0x00017400, 0x00017400, 0x00017400, 0x00017400, + 0x00017400, 0x00017400, 0x00017400, 0x00017400, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_88_avx2_q_low_88_2[] + WC_X64I_UNUSED = { + 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800, + 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_88_avx2_q_2[] + WC_X64I_UNUSED = { + 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4, + 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_88_avx2_44[] + WC_X64I_UNUSED = { + 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c, + 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_88_avx2_vsl[] + WC_X64I_UNUSED = { + 0x0000001f, 0x0000001e, 0x0000001d, 0x0000001c, + 0x0000001b, 0x0000001a, 0x00000019, 0x00000018, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_88_avx2_one[] + WC_X64I_UNUSED = { + 0x00000001, 0x00000001, 0x00000001, 0x00000001, + 0x00000001, 0x00000001, 0x00000001, 0x00000001, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_use_hint_88_avx2(sword32* w1, const byte* h) +{ + word64 rdi, rsi, rax, r8 = 0, rdx = 0, r9, r10, rcx, r11 = 0; + __m256i y0, y1, y2, y3, y4, y5, y6 = _mm256_setzero_si256(), + y7 = _mm256_setzero_si256(), y8, y9, y10, y11, y12, y13, y14; + + rdi = (word64)(size_t)w1; + rsi = (word64)(size_t)h; + + y8 = _mm256_setzero_si256(); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_use_hint_88_avx2_q, + 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_use_hint_88_avx2_q_low_88, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_use_hint_88_avx2_q_low_88_2, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_use_hint_88_avx2_q_2, + 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_use_hint_88_avx2_44, + 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_use_hint_88_avx2_vsl, + 0)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_use_hint_88_avx2_one, + 0)); + rax = (word64)(0); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 0)) & 0xff); + /* 1/4 vectors */ + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 80)) & 0xff); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_0: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_0; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_0; +L_mldsa_use_hint_88_avx2_hints_done_0_0: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_1: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_1; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_1; +L_mldsa_use_hint_88_avx2_hints_done_0_1: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_2: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_2; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_2; +L_mldsa_use_hint_88_avx2_hints_done_0_2: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_3: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_3; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_3; +L_mldsa_use_hint_88_avx2_hints_done_0_3: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_4: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_4; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_4; +L_mldsa_use_hint_88_avx2_hints_done_0_4: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_5: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_5; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_5; +L_mldsa_use_hint_88_avx2_hints_done_0_5: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_6: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_6; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_6; +L_mldsa_use_hint_88_avx2_hints_done_0_6: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_7: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_7; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_7; +L_mldsa_use_hint_88_avx2_hints_done_0_7: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_8: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_8; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_8; +L_mldsa_use_hint_88_avx2_hints_done_0_8: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_9: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_9; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_9; +L_mldsa_use_hint_88_avx2_hints_done_0_9: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_10: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_10; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_10; +L_mldsa_use_hint_88_avx2_hints_done_0_10: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_11: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_11; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_11; +L_mldsa_use_hint_88_avx2_hints_done_0_11: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_12: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_12; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_12; +L_mldsa_use_hint_88_avx2_hints_done_0_12: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_13: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_13; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_13; +L_mldsa_use_hint_88_avx2_hints_done_0_13: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_14: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_14; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_14; +L_mldsa_use_hint_88_avx2_hints_done_0_14: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_0_15: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_0_15; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_0_15; +L_mldsa_use_hint_88_avx2_hints_done_0_15: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y5); + /* 2/4 vectors */ + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 81)) & 0xff); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1024)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1056)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_0: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_0; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_0; +L_mldsa_use_hint_88_avx2_hints_done_1_0: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1024), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1056), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1088)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1120)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_1: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_1; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_1; +L_mldsa_use_hint_88_avx2_hints_done_1_1: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1088), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1120), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1152)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1184)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_2: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_2; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_2; +L_mldsa_use_hint_88_avx2_hints_done_1_2: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1152), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1184), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1216)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1248)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_3: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_3; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_3; +L_mldsa_use_hint_88_avx2_hints_done_1_3: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1216), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1248), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1280)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1312)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_4: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_4; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_4; +L_mldsa_use_hint_88_avx2_hints_done_1_4: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1280), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1312), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1344)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1376)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_5: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_5; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_5; +L_mldsa_use_hint_88_avx2_hints_done_1_5: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1344), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1376), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1408)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1440)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_6: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_6; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_6; +L_mldsa_use_hint_88_avx2_hints_done_1_6: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1408), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1440), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1472)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1504)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_7: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_7; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_7; +L_mldsa_use_hint_88_avx2_hints_done_1_7: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1472), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1504), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1536)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1568)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_8: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_8; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_8; +L_mldsa_use_hint_88_avx2_hints_done_1_8: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1536), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1568), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1600)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1632)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_9: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_9; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_9; +L_mldsa_use_hint_88_avx2_hints_done_1_9: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1600), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1632), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1664)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1696)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_10: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_10; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_10; +L_mldsa_use_hint_88_avx2_hints_done_1_10: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1664), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1696), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1728)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1760)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_11: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_11; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_11; +L_mldsa_use_hint_88_avx2_hints_done_1_11: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1728), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1760), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1792)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1824)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_12: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_12; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_12; +L_mldsa_use_hint_88_avx2_hints_done_1_12: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1792), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1824), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1856)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1888)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_13: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_13; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_13; +L_mldsa_use_hint_88_avx2_hints_done_1_13: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1856), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1888), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1920)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1952)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_14: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_14; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_14; +L_mldsa_use_hint_88_avx2_hints_done_1_14: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1920), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1952), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1984)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2016)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_1_15: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_1_15; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_1_15; +L_mldsa_use_hint_88_avx2_hints_done_1_15: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 1984), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2016), y5); + /* 3/4 vectors */ + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 82)) & 0xff); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2048)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2080)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_0: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_0; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_0; +L_mldsa_use_hint_88_avx2_hints_done_2_0: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2048), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2080), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2112)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2144)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_1: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_1; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_1; +L_mldsa_use_hint_88_avx2_hints_done_2_1: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2112), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2144), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2176)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2208)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_2: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_2; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_2; +L_mldsa_use_hint_88_avx2_hints_done_2_2: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2176), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2208), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2240)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2272)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_3: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_3; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_3; +L_mldsa_use_hint_88_avx2_hints_done_2_3: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2240), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2272), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2304)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2336)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_4: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_4; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_4; +L_mldsa_use_hint_88_avx2_hints_done_2_4: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2304), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2336), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2368)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2400)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_5: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_5; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_5; +L_mldsa_use_hint_88_avx2_hints_done_2_5: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2368), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2400), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2432)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2464)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_6: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_6; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_6; +L_mldsa_use_hint_88_avx2_hints_done_2_6: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2432), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2464), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2496)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2528)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_7: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_7; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_7; +L_mldsa_use_hint_88_avx2_hints_done_2_7: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2496), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2528), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2560)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2592)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_8: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_8; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_8; +L_mldsa_use_hint_88_avx2_hints_done_2_8: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2560), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2592), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2624)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2656)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_9: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_9; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_9; +L_mldsa_use_hint_88_avx2_hints_done_2_9: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2624), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2656), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2688)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2720)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_10: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_10; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_10; +L_mldsa_use_hint_88_avx2_hints_done_2_10: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2688), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2720), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2752)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2784)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_11: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_11; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_11; +L_mldsa_use_hint_88_avx2_hints_done_2_11: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2752), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2784), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2816)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2848)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_12: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_12; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_12; +L_mldsa_use_hint_88_avx2_hints_done_2_12: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2816), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2848), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2880)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2912)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_13: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_13; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_13; +L_mldsa_use_hint_88_avx2_hints_done_2_13: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2880), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2912), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2944)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 2976)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_14: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_14; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_14; +L_mldsa_use_hint_88_avx2_hints_done_2_14: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2944), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 2976), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3008)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3040)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_2_15: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_2_15; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_2_15; +L_mldsa_use_hint_88_avx2_hints_done_2_15: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3008), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3040), y5); + /* 4/4 vectors */ + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 83)) & 0xff); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3072)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3104)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_0: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_0; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_0; +L_mldsa_use_hint_88_avx2_hints_done_3_0: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3072), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3104), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3136)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3168)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_1: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_1; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_1; +L_mldsa_use_hint_88_avx2_hints_done_3_1: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3136), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3168), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3200)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3232)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_2: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_2; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_2; +L_mldsa_use_hint_88_avx2_hints_done_3_2: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3200), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3232), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3264)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3296)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_3: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_3; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_3; +L_mldsa_use_hint_88_avx2_hints_done_3_3: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3264), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3296), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3328)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3360)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_4: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_4; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_4; +L_mldsa_use_hint_88_avx2_hints_done_3_4: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3328), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3360), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3392)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3424)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_5: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_5; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_5; +L_mldsa_use_hint_88_avx2_hints_done_3_5: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3392), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3424), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3456)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3488)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_6: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_6; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_6; +L_mldsa_use_hint_88_avx2_hints_done_3_6: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3456), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3488), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3520)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3552)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_7: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_7; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_7; +L_mldsa_use_hint_88_avx2_hints_done_3_7: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3520), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3552), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3584)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3616)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_8: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_8; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_8; +L_mldsa_use_hint_88_avx2_hints_done_3_8: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3584), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3616), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3648)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3680)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_9: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_9; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_9; +L_mldsa_use_hint_88_avx2_hints_done_3_9: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3648), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3680), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3712)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3744)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_10: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_10; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_10; +L_mldsa_use_hint_88_avx2_hints_done_3_10: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3712), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3744), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3776)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3808)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_11: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_11; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_11; +L_mldsa_use_hint_88_avx2_hints_done_3_11: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3776), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3808), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3840)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3872)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_12: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_12; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_12; +L_mldsa_use_hint_88_avx2_hints_done_3_12: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3840), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3872), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3904)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3936)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_13: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_13; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_13; +L_mldsa_use_hint_88_avx2_hints_done_3_13: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3904), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3936), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 3968)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4000)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_14: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_14; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_14; +L_mldsa_use_hint_88_avx2_hints_done_3_14: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 3968), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 4000), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4032)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 4064)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_mullo_epi32(y0, y12); + y5 = _mm256_mullo_epi32(y1, y12); + y4 = _mm256_add_epi32(y4, y11); + y5 = _mm256_add_epi32(y5, y11); + y4 = _mm256_srli_epi32(y4, 23); + y5 = _mm256_srli_epi32(y5, 23); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_cmpeq_epi32(y4, y12); + y1 = _mm256_cmpeq_epi32(y5, y12); + y2 = _mm256_add_epi32(y2, y0); + y3 = _mm256_add_epi32(y3, y1); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx2_hints_next_3_15: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx2_hints_done_3_15; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx2_hints_next_3_15; +L_mldsa_use_hint_88_avx2_hints_done_3_15: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r10), 0); + r10 = (word64)(r10 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r10), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y0 = _mm256_cmpgt_epi32(y12, y4); + y1 = _mm256_cmpgt_epi32(y12, y5); + y4 = _mm256_and_si256(y4, y0); + y5 = _mm256_and_si256(y5, y1); + y0 = _mm256_srai_epi32(y4, 31); + y1 = _mm256_srai_epi32(y5, 31); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y4 = _mm256_add_epi32(y4, y0); + y5 = _mm256_add_epi32(y5, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 4032), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 4064), y5); +} + +XALIGNED(16) static const word32 L_mldsa_use_hint_32_avx2_q[] WC_X64I_UNUSED = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_32_avx2_q_low_32[] + WC_X64I_UNUSED = { + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_32_avx2_q_low_32_2[] + WC_X64I_UNUSED = { + 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00, + 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_32_avx2_q_low_32_m1[] + WC_X64I_UNUSED = { + 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff, + 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_32_avx2_mask[] + WC_X64I_UNUSED = { + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_32_avx2_vsl[] + WC_X64I_UNUSED = { + 0x0000001f, 0x0000001e, 0x0000001d, 0x0000001c, + 0x0000001b, 0x0000001a, 0x00000019, 0x00000018, +}; + +XALIGNED(16) static const word32 L_mldsa_use_hint_32_avx2_one[] + WC_X64I_UNUSED = { + 0x00000001, 0x00000001, 0x00000001, 0x00000001, + 0x00000001, 0x00000001, 0x00000001, 0x00000001, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_use_hint_32_avx2(sword32* w1, byte k, const byte* h) +{ + word64 rdi, rsi, rdx, r8, r9 = 0, r13, rax = 0, r10 = 0, r11 = 0, rcx = 0, + r12 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), y8, y9, + y10, y11, y12, y13, y14; + + rdi = (word64)(size_t)w1; + rsi = (word64)(word64)k; + rdx = (word64)(size_t)h; + + y8 = _mm256_setzero_si256(); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_use_hint_32_avx2_q, + 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_use_hint_32_avx2_q_low_32, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_use_hint_32_avx2_q_low_32_2, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_use_hint_32_avx2_q_low_32_m1, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_use_hint_32_avx2_mask, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_use_hint_32_avx2_vsl, + 0)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mldsa_use_hint_32_avx2_one, + 0)); + r8 = (word64)(0); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r13 = (word64)(rsi * 10); + r13 = (word64)(r13 - 5); +L_mldsa_use_hint_32_avx2_start_256: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r13)) & 0xff); + r13 = (word64)(r13 + 1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__0: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__0; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__0; +L_mldsa_use_hint_32_avx2_hints_done__0: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__1: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__1; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__1; +L_mldsa_use_hint_32_avx2_hints_done__1: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__2: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__2; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__2; +L_mldsa_use_hint_32_avx2_hints_done__2: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__3: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__3; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__3; +L_mldsa_use_hint_32_avx2_hints_done__3: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__4: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__4; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__4; +L_mldsa_use_hint_32_avx2_hints_done__4: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__5: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__5; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__5; +L_mldsa_use_hint_32_avx2_hints_done__5: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__6: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__6; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__6; +L_mldsa_use_hint_32_avx2_hints_done__6: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__7: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__7; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__7; +L_mldsa_use_hint_32_avx2_hints_done__7: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__8: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__8; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__8; +L_mldsa_use_hint_32_avx2_hints_done__8: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__9: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__9; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__9; +L_mldsa_use_hint_32_avx2_hints_done__9: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__10: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__10; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__10; +L_mldsa_use_hint_32_avx2_hints_done__10: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__11: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__11; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__11; +L_mldsa_use_hint_32_avx2_hints_done__11: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__12: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__12; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__12; +L_mldsa_use_hint_32_avx2_hints_done__12: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__13: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__13; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__13; +L_mldsa_use_hint_32_avx2_hints_done__13: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__14: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__14; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__14; +L_mldsa_use_hint_32_avx2_hints_done__14: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y2 = _mm256_srai_epi32(y0, 31); + y3 = _mm256_srai_epi32(y1, 31); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y4 = _mm256_add_epi32(y0, y11); + y5 = _mm256_add_epi32(y1, y11); + y4 = _mm256_srli_epi32(y4, 19); + y5 = _mm256_srli_epi32(y5, 19); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_sub_epi32(y9, y2); + y3 = _mm256_sub_epi32(y9, y3); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y2 = _mm256_mullo_epi32(y4, y10); + y3 = _mm256_mullo_epi32(y5, y10); + y2 = _mm256_sub_epi32(y0, y2); + y3 = _mm256_sub_epi32(y1, y3); + y0 = _mm256_srli_epi32(y4, 4); + y1 = _mm256_srli_epi32(y5, 4); + y2 = _mm256_sub_epi32(y2, y0); + y3 = _mm256_sub_epi32(y3, y1); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx2_hints_next__15: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx2_hints_done__15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx2_hints_done__15; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx2_hints_next__15; +L_mldsa_use_hint_32_avx2_hints_done__15: + y6 = _mm256_inserti128_si256(y6, _mm_cvtsi32_si128((int)(word32)r11), 0); + r11 = (word64)(r11 >> 8); + y7 = _mm256_inserti128_si256(y7, _mm_cvtsi32_si128((int)(word32)r11), 0); + y6 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y6)); + y7 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y7)); + y6 = _mm256_sllv_epi32(y6, y13); + y7 = _mm256_sllv_epi32(y7, y13); + y6 = _mm256_srai_epi32(y6, 31); + y7 = _mm256_srai_epi32(y7, 31); + y2 = _mm256_sub_epi32(y2, y14); + y3 = _mm256_sub_epi32(y3, y14); + y2 = _mm256_srli_epi32(y2, 31); + y3 = _mm256_srli_epi32(y3, 31); + y2 = _mm256_slli_epi32(y2, 1); + y3 = _mm256_slli_epi32(y3, 1); + y2 = _mm256_sub_epi32(y14, y2); + y3 = _mm256_sub_epi32(y14, y3); + y2 = _mm256_and_si256(y2, y6); + y3 = _mm256_and_si256(y3, y7); + y4 = _mm256_add_epi32(y4, y2); + y5 = _mm256_add_epi32(y5, y3); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y5); + rdi = (word64)(rdi + 0x400); + rsi = (word64)(rsi - 1); + if ((rsi) != (0)) { + goto L_mldsa_use_hint_32_avx2_start_256; + } +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL int wc_mldsa_vec_check_low_avx2(const sword32* a, byte l, + sword32 hi) +{ + word64 rdi, rsi, rdx, rax = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(); + word64 zf1; + word64 zf2; + word64 zf3; + word64 zf4; + word64 zf5; + word64 zf6; + word64 zf7; + word64 zf8; + word64 zf9; + word64 zf10; + word64 zf11; + word64 zf12; + word64 zf13; + word64 zf14; + word64 zf15; + word64 zf16; + word64 zf17; + word64 zf18; + word64 zf19; + word64 zf20; + word64 zf21; + word64 zf22; + word64 zf23; + word64 zf24; + word64 zf25; + word64 zf26; + word64 zf27; + word64 zf28; + word64 zf29; + word64 zf30; + word64 zf31; + word64 zf32; + + rdi = (word64)(size_t)a; + rsi = (word64)(byte)l; + rdx = (word64)(word32)hi; + + rdx = (word32)((word32)rdx - 1); + y2 = _mm256_inserti128_si256(y2, _mm_cvtsi32_si128((int)(word32)rdx), 0); + rdx = (word32)(0 - (word32)rdx); + y3 = _mm256_inserti128_si256(y3, _mm_cvtsi32_si128((int)(word32)rdx), 0); + y2 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y2)); + y3 = _mm256_broadcastd_epi32(_mm256_castsi256_si128(y3)); +L_mldsa_vec_check_low_vx2_start_256: + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf1 = rax; + zf2 = 0; + rax = (word64)(0); + if ((zf1) != (zf2)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf3 = rax; + zf4 = 0; + rax = (word64)(0); + if ((zf3) != (zf4)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf5 = rax; + zf6 = 0; + rax = (word64)(0); + if ((zf5) != (zf6)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf7 = rax; + zf8 = 0; + rax = (word64)(0); + if ((zf7) != (zf8)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf9 = rax; + zf10 = 0; + rax = (word64)(0); + if ((zf9) != (zf10)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf11 = rax; + zf12 = 0; + rax = (word64)(0); + if ((zf11) != (zf12)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf13 = rax; + zf14 = 0; + rax = (word64)(0); + if ((zf13) != (zf14)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf15 = rax; + zf16 = 0; + rax = (word64)(0); + if ((zf15) != (zf16)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf17 = rax; + zf18 = 0; + rax = (word64)(0); + if ((zf17) != (zf18)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf19 = rax; + zf20 = 0; + rax = (word64)(0); + if ((zf19) != (zf20)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf21 = rax; + zf22 = 0; + rax = (word64)(0); + if ((zf21) != (zf22)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf23 = rax; + zf24 = 0; + rax = (word64)(0); + if ((zf23) != (zf24)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf25 = rax; + zf26 = 0; + rax = (word64)(0); + if ((zf25) != (zf26)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf27 = rax; + zf28 = 0; + rax = (word64)(0); + if ((zf27) != (zf28)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf29 = rax; + zf30 = 0; + rax = (word64)(0); + if ((zf29) != (zf30)) { + goto L_mldsa_vec_check_low_vx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y4 = _mm256_cmpgt_epi32(y0, y2); + y5 = _mm256_cmpgt_epi32(y1, y2); + y6 = _mm256_cmpgt_epi32(y3, y0); + y7 = _mm256_cmpgt_epi32(y3, y1); + y4 = _mm256_or_si256(y4, y5); + y6 = _mm256_or_si256(y6, y7); + y4 = _mm256_or_si256(y4, y6); + rax = (word32)((word32)_mm256_movemask_epi8(y4)); + zf31 = rax; + zf32 = 0; + rax = (word64)(0); + if ((zf31) != (zf32)) { + goto L_mldsa_vec_check_low_vx2_done; + } + rdi = (word64)(rdi + 0x400); + rsi = (word64)(rsi - 1); + if ((rsi) != (0)) { + goto L_mldsa_vec_check_low_vx2_start_256; + } + rax = (word64)(1); +L_mldsa_vec_check_low_vx2_done: + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_poly_add_avx2(sword32* r, const sword32* a) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y0 = _mm256_add_epi32(y0, y8); + y1 = _mm256_add_epi32(y1, y9); + y2 = _mm256_add_epi32(y2, y10); + y3 = _mm256_add_epi32(y3, y11); + y4 = _mm256_add_epi32(y4, y12); + y5 = _mm256_add_epi32(y5, y13); + y6 = _mm256_add_epi32(y6, y14); + y7 = _mm256_add_epi32(y7, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y0 = _mm256_add_epi32(y0, y8); + y1 = _mm256_add_epi32(y1, y9); + y2 = _mm256_add_epi32(y2, y10); + y3 = _mm256_add_epi32(y3, y11); + y4 = _mm256_add_epi32(y4, y12); + y5 = _mm256_add_epi32(y5, y13); + y6 = _mm256_add_epi32(y6, y14); + y7 = _mm256_add_epi32(y7, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 512)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 544)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 576)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 608)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 640)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 672)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 704)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 736)); + y0 = _mm256_add_epi32(y0, y8); + y1 = _mm256_add_epi32(y1, y9); + y2 = _mm256_add_epi32(y2, y10); + y3 = _mm256_add_epi32(y3, y11); + y4 = _mm256_add_epi32(y4, y12); + y5 = _mm256_add_epi32(y5, y13); + y6 = _mm256_add_epi32(y6, y14); + y7 = _mm256_add_epi32(y7, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 768)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 800)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 832)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 896)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 928)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 960)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 992)); + y0 = _mm256_add_epi32(y0, y8); + y1 = _mm256_add_epi32(y1, y9); + y2 = _mm256_add_epi32(y2, y10); + y3 = _mm256_add_epi32(y3, y11); + y4 = _mm256_add_epi32(y4, y12); + y5 = _mm256_add_epi32(y5, y13); + y6 = _mm256_add_epi32(y6, y14); + y7 = _mm256_add_epi32(y7, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_poly_sub_avx2(sword32* r, const sword32* a) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y0 = _mm256_sub_epi32(y0, y8); + y1 = _mm256_sub_epi32(y1, y9); + y2 = _mm256_sub_epi32(y2, y10); + y3 = _mm256_sub_epi32(y3, y11); + y4 = _mm256_sub_epi32(y4, y12); + y5 = _mm256_sub_epi32(y5, y13); + y6 = _mm256_sub_epi32(y6, y14); + y7 = _mm256_sub_epi32(y7, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y0 = _mm256_sub_epi32(y0, y8); + y1 = _mm256_sub_epi32(y1, y9); + y2 = _mm256_sub_epi32(y2, y10); + y3 = _mm256_sub_epi32(y3, y11); + y4 = _mm256_sub_epi32(y4, y12); + y5 = _mm256_sub_epi32(y5, y13); + y6 = _mm256_sub_epi32(y6, y14); + y7 = _mm256_sub_epi32(y7, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 512)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 544)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 576)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 608)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 640)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 672)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 704)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 736)); + y0 = _mm256_sub_epi32(y0, y8); + y1 = _mm256_sub_epi32(y1, y9); + y2 = _mm256_sub_epi32(y2, y10); + y3 = _mm256_sub_epi32(y3, y11); + y4 = _mm256_sub_epi32(y4, y12); + y5 = _mm256_sub_epi32(y5, y13); + y6 = _mm256_sub_epi32(y6, y14); + y7 = _mm256_sub_epi32(y7, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 768)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 800)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 832)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 864)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 896)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 928)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 960)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 992)); + y0 = _mm256_sub_epi32(y0, y8); + y1 = _mm256_sub_epi32(y1, y9); + y2 = _mm256_sub_epi32(y2, y10); + y3 = _mm256_sub_epi32(y3, y11); + y4 = _mm256_sub_epi32(y4, y12); + y5 = _mm256_sub_epi32(y5, y13); + y6 = _mm256_sub_epi32(y6, y14); + y7 = _mm256_sub_epi32(y7, y15); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void wc_mldsa_poly_make_pos_avx2(sword32* a) +{ + word64 rdi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9; + + rdi = (word64)(size_t)a; + + y8 = _mm256_setzero_si256(); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(mldsa_q, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_cmpgt_epi32(y8, y0); + y5 = _mm256_cmpgt_epi32(y8, y1); + y6 = _mm256_cmpgt_epi32(y8, y2); + y7 = _mm256_cmpgt_epi32(y8, y3); + y4 = _mm256_and_si256(y4, y9); + y5 = _mm256_and_si256(y5, y9); + y6 = _mm256_and_si256(y6, y9); + y7 = _mm256_and_si256(y7, y9); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y4 = _mm256_cmpgt_epi32(y8, y0); + y5 = _mm256_cmpgt_epi32(y8, y1); + y6 = _mm256_cmpgt_epi32(y8, y2); + y7 = _mm256_cmpgt_epi32(y8, y3); + y4 = _mm256_and_si256(y4, y9); + y5 = _mm256_and_si256(y5, y9); + y6 = _mm256_and_si256(y6, y9); + y7 = _mm256_and_si256(y7, y9); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_cmpgt_epi32(y8, y0); + y5 = _mm256_cmpgt_epi32(y8, y1); + y6 = _mm256_cmpgt_epi32(y8, y2); + y7 = _mm256_cmpgt_epi32(y8, y3); + y4 = _mm256_and_si256(y4, y9); + y5 = _mm256_and_si256(y5, y9); + y6 = _mm256_and_si256(y6, y9); + y7 = _mm256_and_si256(y7, y9); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_cmpgt_epi32(y8, y0); + y5 = _mm256_cmpgt_epi32(y8, y1); + y6 = _mm256_cmpgt_epi32(y8, y2); + y7 = _mm256_cmpgt_epi32(y8, y3); + y4 = _mm256_and_si256(y4, y9); + y5 = _mm256_and_si256(y5, y9); + y6 = _mm256_and_si256(y6, y9); + y7 = _mm256_and_si256(y7, y9); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y4 = _mm256_cmpgt_epi32(y8, y0); + y5 = _mm256_cmpgt_epi32(y8, y1); + y6 = _mm256_cmpgt_epi32(y8, y2); + y7 = _mm256_cmpgt_epi32(y8, y3); + y4 = _mm256_and_si256(y4, y9); + y5 = _mm256_and_si256(y5, y9); + y6 = _mm256_and_si256(y6, y9); + y7 = _mm256_and_si256(y7, y9); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 512), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 544), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 576), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 608), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y4 = _mm256_cmpgt_epi32(y8, y0); + y5 = _mm256_cmpgt_epi32(y8, y1); + y6 = _mm256_cmpgt_epi32(y8, y2); + y7 = _mm256_cmpgt_epi32(y8, y3); + y4 = _mm256_and_si256(y4, y9); + y5 = _mm256_and_si256(y5, y9); + y6 = _mm256_and_si256(y6, y9); + y7 = _mm256_and_si256(y7, y9); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 640), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 672), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 704), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 736), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y4 = _mm256_cmpgt_epi32(y8, y0); + y5 = _mm256_cmpgt_epi32(y8, y1); + y6 = _mm256_cmpgt_epi32(y8, y2); + y7 = _mm256_cmpgt_epi32(y8, y3); + y4 = _mm256_and_si256(y4, y9); + y5 = _mm256_and_si256(y5, y9); + y6 = _mm256_and_si256(y6, y9); + y7 = _mm256_and_si256(y7, y9); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 768), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 800), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 832), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 864), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y4 = _mm256_cmpgt_epi32(y8, y0); + y5 = _mm256_cmpgt_epi32(y8, y1); + y6 = _mm256_cmpgt_epi32(y8, y2); + y7 = _mm256_cmpgt_epi32(y8, y3); + y4 = _mm256_and_si256(y4, y9); + y5 = _mm256_and_si256(y5, y9); + y6 = _mm256_and_si256(y6, y9); + y7 = _mm256_and_si256(y7, y9); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_add_epi32(y2, y6); + y3 = _mm256_add_epi32(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 896), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 928), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 960), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 992), y3); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* WOLFSSL_HAVE_MLDSA */ +#ifdef WOLFSSL_HAVE_MLDSA +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifdef HAVE_INTEL_AVX512 +XALIGNED(32) static const word64 L_mldsa_avx512_perm20[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000008ULL, 0x0000000000000009ULL, + 0x0000000000000004ULL, 0x0000000000000005ULL, + 0x000000000000000cULL, 0x000000000000000dULL, +}; + +XALIGNED(32) static const word64 L_mldsa_avx512_perm31[] WC_X64I_UNUSED = { + 0x0000000000000002ULL, 0x0000000000000003ULL, + 0x000000000000000aULL, 0x000000000000000bULL, + 0x0000000000000006ULL, 0x0000000000000007ULL, + 0x000000000000000eULL, 0x000000000000000fULL, +}; + +XALIGNED(32) static const word64 L_mldsa_avx512_perm21[] WC_X64I_UNUSED = { + 0x0000000000000002ULL, 0x0000000000000003ULL, + 0x0000000000000008ULL, 0x0000000000000009ULL, + 0x0000000000000006ULL, 0x0000000000000007ULL, + 0x000000000000000cULL, 0x000000000000000dULL, +}; + +XALIGNED(32) static const word64 L_mldsa_avx512_permn0[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000008ULL, 0x0000000000000009ULL, + 0x0000000000000002ULL, 0x0000000000000003ULL, + 0x000000000000000aULL, 0x000000000000000bULL, +}; + +XALIGNED(32) static const word64 L_mldsa_avx512_permn1[] WC_X64I_UNUSED = { + 0x0000000000000004ULL, 0x0000000000000005ULL, + 0x000000000000000cULL, 0x000000000000000dULL, + 0x0000000000000006ULL, 0x0000000000000007ULL, + 0x000000000000000eULL, 0x000000000000000fULL, +}; + +XALIGNED(16) static const word32 L_mldsa_rej_q_avx512[] WC_X64I_UNUSED = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +XALIGNED(32) static const word64 L_mldsa_rej_mask_avx512[] WC_X64I_UNUSED = { + 0x007fffff007fffffULL, 0x007fffff007fffffULL, + 0x007fffff007fffffULL, 0x007fffff007fffffULL, +}; + +XALIGNED(32) static const word64 L_mldsa_rej_shuffle_avx512[] WC_X64I_UNUSED = { + 0x0005040300020100ULL, 0x000b0a0900080706ULL, + 0x0009080700060504ULL, 0x000f0e0d000c0b0aULL, +}; + +WC_X64I_TARGET("avx,avx512f,avx512vl,avx512bw,bmi,bmi2") +WOLFSSL_LOCAL int wc_mldsa_rej_uniform_n_avx512(sword32* p, word32 len, + const byte* r, word32 rLen) +{ + word64 rdi, rsi, rdx, r8, rax, rbx = 0, r10 = 0, r9 = 0, rcx = 0; + __m512i z0 = _mm512_setzero_si512(), z1, z2, z3; + __mmask16 k1; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + r8 = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + z1 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_rej_q_avx512, 0))); + z2 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_rej_mask_avx512, 0))); + z3 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_rej_shuffle_avx512, 0))); +L_wc_mldsa_rej_uniform_n_avx512_start_512: + if ((sword32)((word32)rsi) < (sword32)(0x10)) { + goto L_wc_mldsa_rej_uniform_n_avx512_done_512; + } + if ((sword32)((word32)r8) < (sword32)(0x38)) { + goto L_wc_mldsa_rej_uniform_n_avx512_done_512; + } + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 24)), 1); + z0 = _mm512_permutex_epi64(z0, 0x94); + z0 = _mm512_shuffle_epi8(z0, z3); + z0 = _mm512_and_si512(z0, z2); + k1 = _mm512_cmp_epi32_mask(z0, z1, 1); + _mm512_mask_compressstoreu_epi32(WC_PW(rdi, 0), k1, z0); + rbx = (word32)((word32)k1); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 4); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + r8 = (word32)((word32)r8 - 0x30); + goto L_wc_mldsa_rej_uniform_n_avx512_start_512; +L_wc_mldsa_rej_uniform_n_avx512_done_512: + if (((word32)rsi) == (0)) { + goto L_wc_mldsa_rej_uniform_n_avx512_done_64; + } + r10 = (word64)(0xffffff00ffffff); + r9 = (word64)(0x7fffff007fffff); +L_wc_mldsa_rej_uniform_n_avx512_start_64: + if ((sword32)((word32)r8) < (sword32)(8)) { + goto L_wc_mldsa_rej_uniform_n_avx512_done_64; + } + rcx = (word64)(WC_L64(rdx, 0)); + rcx = (word64)((word64)_pdep_u64(rcx, r10)); + rcx = (word64)(rcx & r9); + if ((sword32)((word32)rcx) >= (sword32)(0x7fe001)) { + goto L_wc_mldsa_rej_uniform_n_avx512_rej_large_0; + } + WC_S32(rdi, 0) = (word32)((word32)rcx); + rdi = (word64)(rdi + 4); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_wc_mldsa_rej_uniform_n_avx512_done_64; + } +L_wc_mldsa_rej_uniform_n_avx512_rej_large_0: + rcx = (word64)(rcx >> 32); + if ((sword32)((word32)rcx) >= (sword32)(0x7fe001)) { + goto L_wc_mldsa_rej_uniform_n_avx512_rej_large_1; + } + WC_S32(rdi, 0) = (word32)((word32)rcx); + rdi = (word64)(rdi + 4); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_wc_mldsa_rej_uniform_n_avx512_done_64; + } +L_wc_mldsa_rej_uniform_n_avx512_rej_large_1: + rdx = (word64)(rdx + 6); + r8 = (word32)((word32)r8 - 6); + if ((sword32)((word32)r8) <= (sword32)(0)) { + goto L_wc_mldsa_rej_uniform_n_avx512_done_64; + } + if ((sword32)((word32)rsi) > (sword32)(0)) { + goto L_wc_mldsa_rej_uniform_n_avx512_start_64; + } +L_wc_mldsa_rej_uniform_n_avx512_done_64: + rax = (word32)((word32)rax - (word32)rsi); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx,avx512f,avx512vl,avx512bw,bmi,bmi2") +WOLFSSL_LOCAL int wc_mldsa_rej_uniform_avx512(sword32* p, word32 len, + const byte* r, word32 rLen) +{ + word64 rdi, rsi, rdx, r8, rax, rbx = 0, r10 = 0, r9 = 0, rcx = 0; + __m512i z0 = _mm512_setzero_si512(), z1, z2, z3; + __mmask16 k1; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + r8 = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + z1 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_rej_q_avx512, 0))); + z2 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_rej_mask_avx512, 0))); + z3 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_rej_shuffle_avx512, 0))); +L_wc_mldsa_rej_uniform_avx512_start_512: + if ((sword32)((word32)rsi) < (sword32)(0x10)) { + goto L_wc_mldsa_rej_uniform_avx512_done_512; + } + if ((sword32)((word32)r8) < (sword32)(0x38)) { + goto L_wc_mldsa_rej_uniform_avx512_done_512; + } + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 24)), 1); + z0 = _mm512_permutex_epi64(z0, 0x94); + z0 = _mm512_shuffle_epi8(z0, z3); + z0 = _mm512_and_si512(z0, z2); + k1 = _mm512_cmp_epi32_mask(z0, z1, 1); + _mm512_mask_compressstoreu_epi32(WC_PW(rdi, 0), k1, z0); + rbx = (word32)((word32)k1); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 4); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + r8 = (word32)((word32)r8 - 0x30); + goto L_wc_mldsa_rej_uniform_avx512_start_512; +L_wc_mldsa_rej_uniform_avx512_done_512: + if (((word32)rsi) == (0)) { + goto L_wc_mldsa_rej_uniform_avx512_done_64; + } + r10 = (word64)(0xffffff00ffffff); + r9 = (word64)(0x7fffff007fffff); +L_wc_mldsa_rej_uniform_avx512_start_64: + if ((sword32)((word32)r8) < (sword32)(8)) { + goto L_wc_mldsa_rej_uniform_avx512_done_64; + } + rcx = (word64)(WC_L64(rdx, 0)); + rcx = (word64)((word64)_pdep_u64(rcx, r10)); + rcx = (word64)(rcx & r9); + if ((sword32)((word32)rcx) >= (sword32)(0x7fe001)) { + goto L_wc_mldsa_rej_uniform_avx512_rej_large_0; + } + WC_S32(rdi, 0) = (word32)((word32)rcx); + rdi = (word64)(rdi + 4); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_wc_mldsa_rej_uniform_avx512_done_64; + } +L_wc_mldsa_rej_uniform_avx512_rej_large_0: + rcx = (word64)(rcx >> 32); + if ((sword32)((word32)rcx) >= (sword32)(0x7fe001)) { + goto L_wc_mldsa_rej_uniform_avx512_rej_large_1; + } + WC_S32(rdi, 0) = (word32)((word32)rcx); + rdi = (word64)(rdi + 4); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_wc_mldsa_rej_uniform_avx512_done_64; + } +L_wc_mldsa_rej_uniform_avx512_rej_large_1: + rdx = (word64)(rdx + 6); + r8 = (word32)((word32)r8 - 6); + if ((sword32)((word32)r8) <= (sword32)(0)) { + goto L_wc_mldsa_rej_uniform_avx512_done_64; + } + if ((sword32)((word32)rsi) > (sword32)(0)) { + goto L_wc_mldsa_rej_uniform_avx512_start_64; + } +L_wc_mldsa_rej_uniform_avx512_done_64: + rax = (word32)((word32)rax - (word32)rsi); + return (int)(word32)rax; +} + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta2_avx512_over[] + WC_X64I_UNUSED = { + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, +}; + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta2_avx512_mask_nibbles[] + WC_X64I_UNUSED = { + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, +}; + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta2_avx512_table[] + WC_X64I_UNUSED = { + 0x00000002, 0x00000001, 0x00000000, 0xffffffff, + 0xfffffffe, 0x00000002, 0x00000001, 0x00000000, + 0xffffffff, 0xfffffffe, 0x00000002, 0x00000001, + 0x00000000, 0xffffffff, 0xfffffffe, 0x00000000, +}; + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta2_avx512_dup[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000000, 0x00000001, 0x00000001, + 0x00000002, 0x00000002, 0x00000003, 0x00000003, + 0x00000004, 0x00000004, 0x00000005, 0x00000005, + 0x00000006, 0x00000006, 0x00000007, 0x00000007, +}; + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta2_avx512_shift[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000000, 0x00000004, 0x00000000, 0x00000004, +}; + +WC_X64I_TARGET("avx512f,bmi") +WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta2_avx512(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt) +{ + word64 rdi, rsi, rdx, rcx, r11, r8, rax = 0, rbx = 0, r9 = 0, r10 = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), z2, z3, + z4, z5, z6; + __mmask16 k1; + unsigned int ac1; + unsigned int ac2; + unsigned char cf; + + rdi = (word64)(size_t)z; + rsi = (word64)(word32)zLen; + rdx = (word64)(size_t)s; + rcx = (word64)(size_t)cnt; + + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta2_avx512_dup, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta2_avx512_shift, 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta2_avx512_mask_nibbles, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta2_avx512_over, 0)); + z6 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta2_avx512_table, 0)); + r11 = (word64)((word64)(size_t)L_mldsa_extract_coeffs_eta2_avx512_table); + r8 = (word32)(WC_L32(rcx, 0)); + if (((word32)r8) != (0)) { + goto L_mldsa_extract_coeffs_eta2_avx512_less_than_256; + } + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 0)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 8)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 16)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 24)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 32)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 40)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 48)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 56)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 64)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 72)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 80)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 88)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 96)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 104)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 112)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 120)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + rsi = (word32)((word32)rsi - 0x80); + rdi = (word64)(rdi + 0x80); +L_mldsa_extract_coeffs_eta2_avx512_less_than_256: + if ((sword32)((word32)r8) > (sword32)(0xf0)) { + goto L_mldsa_extract_coeffs_eta2_avx512_less_than_zmm; + } +L_mldsa_extract_coeffs_eta2_avx512_start_one_zmm: + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 0)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + rsi = (word32)((word32)rsi - 8); + rdi = (word64)(rdi + 8); + if ((sword32)((word32)rsi) < (sword32)(8)) { + goto L_mldsa_extract_coeffs_eta2_avx512_less_than_zmm; + } + if ((sword32)((word32)r8) <= (sword32)(0xf0)) { + goto L_mldsa_extract_coeffs_eta2_avx512_start_one_zmm; + } +L_mldsa_extract_coeffs_eta2_avx512_less_than_zmm: + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta2_avx512_done; + } +L_mldsa_extract_coeffs_eta2_avx512_start_byte: + if (((word32)rsi) == (0)) { + goto L_mldsa_extract_coeffs_eta2_avx512_done; + } + rbx = (word32)((word32)WC_L8(rdi, 0)); + rdi = (word64)(rdi + 1); + rsi = (word32)((word32)rsi - 1); + rax = (word32)((word32)rbx); + rax = (word32)((word32)rax >> 4); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)((byte)rbx & 0xf) & 0xff); + r9 = (word64)(0); + cf = _addcarry_u32((unsigned char)(((byte)rbx) < (0xf)), (word32)(word32)r9, + (word32)0, &ac1); + r9 = (word32)ac1; + r10 = (word32)(WC_L32(r11, rbx * 4)); + WC_S32(rdx, 0) = (word32)((word32)r10); + r8 = (word32)((word32)r8 + (word32)r9); + r9 = (word32)((word32)r9 << 2); + rdx = (word64)(rdx + r9); + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta2_avx512_done; + } + r9 = (word64)(0); + cf = _addcarry_u32((unsigned char)(((byte)rax) < (0xf)), (word32)(word32)r9, + (word32)0, &ac2); + r9 = (word32)ac2; + r10 = (word32)(WC_L32(r11, rax * 4)); + WC_S32(rdx, 0) = (word32)((word32)r10); + r8 = (word32)((word32)r8 + (word32)r9); + r9 = (word32)((word32)r9 << 2); + rdx = (word64)(rdx + r9); + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta2_avx512_done; + } + goto L_mldsa_extract_coeffs_eta2_avx512_start_byte; +L_mldsa_extract_coeffs_eta2_avx512_done: + WC_S32(rcx, 0) = (word32)((word32)r8); + (void)cf; +} + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta4_avx512_over[] + WC_X64I_UNUSED = { + 0x00000009, 0x00000009, 0x00000009, 0x00000009, + 0x00000009, 0x00000009, 0x00000009, 0x00000009, + 0x00000009, 0x00000009, 0x00000009, 0x00000009, + 0x00000009, 0x00000009, 0x00000009, 0x00000009, +}; + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta4_avx512_mask_nibbles[] + WC_X64I_UNUSED = { + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, +}; + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta4_avx512_table[] + WC_X64I_UNUSED = { + 0x00000004, 0x00000003, 0x00000002, 0x00000001, + 0x00000000, 0xffffffff, 0xfffffffe, 0xfffffffd, + 0xfffffffc, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta4_avx512_dup[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000000, 0x00000001, 0x00000001, + 0x00000002, 0x00000002, 0x00000003, 0x00000003, + 0x00000004, 0x00000004, 0x00000005, 0x00000005, + 0x00000006, 0x00000006, 0x00000007, 0x00000007, +}; + +XALIGNED(16) static const word32 L_mldsa_extract_coeffs_eta4_avx512_shift[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000000, 0x00000004, 0x00000000, 0x00000004, +}; + +WC_X64I_TARGET("avx512f,bmi") +WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta4_avx512(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt) +{ + word64 rdi, rsi, rdx, rcx, r11, r8, rax = 0, rbx = 0, r9 = 0, r10 = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), z2, z3, + z4, z5, z6; + __mmask16 k1; + unsigned int ac1; + unsigned int ac2; + unsigned char cf; + + rdi = (word64)(size_t)z; + rsi = (word64)(word32)zLen; + rdx = (word64)(size_t)s; + rcx = (word64)(size_t)cnt; + + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta4_avx512_dup, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta4_avx512_shift, 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta4_avx512_mask_nibbles, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta4_avx512_over, 0)); + z6 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_extract_coeffs_eta4_avx512_table, 0)); + r11 = (word64)((word64)(size_t)L_mldsa_extract_coeffs_eta4_avx512_table); + r8 = (word32)(WC_L32(rcx, 0)); + if (((word32)r8) != (0)) { + goto L_mldsa_extract_coeffs_eta4_avx512_less_than_256; + } + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 0)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 8)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 16)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 24)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 32)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 40)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 48)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 56)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 64)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 72)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 80)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 88)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 96)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 104)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 112)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + z1 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 120)))); + z1 = _mm512_permutexvar_epi32(z2, z1); + z1 = _mm512_srlv_epi32(z1, z3); + z1 = _mm512_and_si512(z1, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + k1 = _mm512_cmp_epi32_mask(z1, z5, 1); + z1 = _mm512_permutexvar_epi32(z1, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z1); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + rsi = (word32)((word32)rsi - 0x80); + rdi = (word64)(rdi + 0x80); +L_mldsa_extract_coeffs_eta4_avx512_less_than_256: + if ((sword32)((word32)r8) > (sword32)(0xf0)) { + goto L_mldsa_extract_coeffs_eta4_avx512_less_than_zmm; + } +L_mldsa_extract_coeffs_eta4_avx512_start_one_zmm: + z0 = _mm512_zextsi256_si512(_mm256_cvtepu8_epi32(_mm_loadl_epi64(( + const __m128i*)WC_PR(rdi, 0)))); + z0 = _mm512_permutexvar_epi32(z2, z0); + z0 = _mm512_srlv_epi32(z0, z3); + z0 = _mm512_and_si512(z0, z4); + k1 = _mm512_cmp_epi32_mask(z0, z5, 1); + z0 = _mm512_permutexvar_epi32(z0, z6); + _mm512_mask_compressstoreu_epi32(WC_PW(rdx, 0), k1, z0); + rax = (word32)((word32)k1); + rax = (word32)(WC_X64I_POPCNT32((word32)rax)); + rdx = (word64)(rdx + rax * 4); + r8 = (word32)((word32)r8 + (word32)rax); + rsi = (word32)((word32)rsi - 8); + rdi = (word64)(rdi + 8); + if ((sword32)((word32)rsi) < (sword32)(8)) { + goto L_mldsa_extract_coeffs_eta4_avx512_less_than_zmm; + } + if ((sword32)((word32)r8) <= (sword32)(0xf0)) { + goto L_mldsa_extract_coeffs_eta4_avx512_start_one_zmm; + } +L_mldsa_extract_coeffs_eta4_avx512_less_than_zmm: + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta4_avx512_done; + } +L_mldsa_extract_coeffs_eta4_avx512_start_byte: + if (((word32)rsi) == (0)) { + goto L_mldsa_extract_coeffs_eta4_avx512_done; + } + rbx = (word32)((word32)WC_L8(rdi, 0)); + rdi = (word64)(rdi + 1); + rsi = (word32)((word32)rsi - 1); + rax = (word32)((word32)rbx); + rax = (word32)((word32)rax >> 4); + rbx = (rbx & ~(word64)0xff) | ((word64)(byte)((byte)rbx & 0xf) & 0xff); + r9 = (word64)(0); + cf = _addcarry_u32((unsigned char)(((byte)rbx) < (9)), (word32)(word32)r9, ( + word32)0, &ac1); + r9 = (word32)ac1; + r10 = (word32)(WC_L32(r11, rbx * 4)); + WC_S32(rdx, 0) = (word32)((word32)r10); + r8 = (word32)((word32)r8 + (word32)r9); + r9 = (word32)((word32)r9 << 2); + rdx = (word64)(rdx + r9); + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta4_avx512_done; + } + r9 = (word64)(0); + cf = _addcarry_u32((unsigned char)(((byte)rax) < (9)), (word32)(word32)r9, ( + word32)0, &ac2); + r9 = (word32)ac2; + r10 = (word32)(WC_L32(r11, rax * 4)); + WC_S32(rdx, 0) = (word32)((word32)r10); + r8 = (word32)((word32)r8 + (word32)r9); + r9 = (word32)((word32)r9 << 2); + rdx = (word64)(rdx + r9); + if (((word32)r8) == (0x100)) { + goto L_mldsa_extract_coeffs_eta4_avx512_done; + } + goto L_mldsa_extract_coeffs_eta4_avx512_start_byte; +L_mldsa_extract_coeffs_eta4_avx512_done: + WC_S32(rcx, 0) = (word32)((word32)r8); + (void)cf; +} + +XALIGNED(16) static const word32 L_mldsa_make_hint_88_avx512_low[] + WC_X64I_UNUSED = { + 0x00017400, 0x00017400, 0x00017400, 0x00017400, + 0x00017400, 0x00017400, 0x00017400, 0x00017400, + 0x00017400, 0x00017400, 0x00017400, 0x00017400, + 0x00017400, 0x00017400, 0x00017400, 0x00017400, +}; + +XALIGNED(16) static const word32 L_mldsa_make_hint_88_avx512_neg_low[] + WC_X64I_UNUSED = { + 0xfffe8c00, 0xfffe8c00, 0xfffe8c00, 0xfffe8c00, + 0xfffe8c00, 0xfffe8c00, 0xfffe8c00, 0xfffe8c00, + 0xfffe8c00, 0xfffe8c00, 0xfffe8c00, 0xfffe8c00, + 0xfffe8c00, 0xfffe8c00, 0xfffe8c00, 0xfffe8c00, +}; + +XALIGNED(16) static const word32 L_mldsa_make_hint_88_avx512_seq[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000002, 0x00000003, + 0x00000004, 0x00000005, 0x00000006, 0x00000007, + 0x00000008, 0x00000009, 0x0000000a, 0x0000000b, + 0x0000000c, 0x0000000d, 0x0000000e, 0x0000000f, +}; + +XALIGNED(16) static const word32 L_mldsa_make_hint_88_avx512_step[] + WC_X64I_UNUSED = { + 0x00000010, 0x00000010, 0x00000010, 0x00000010, + 0x00000010, 0x00000010, 0x00000010, 0x00000010, + 0x00000010, 0x00000010, 0x00000010, 0x00000010, + 0x00000010, 0x00000010, 0x00000010, 0x00000010, +}; + +WC_X64I_TARGET("avx,avx512f,bmi,bmi2") +WOLFSSL_LOCAL int wc_mldsa_make_hint_88_avx512(const sword32* s, + const sword32* w1, unsigned int omega, byte* h, byte* idx) +{ + word64 rdi, rsi, rdx, rcx, r8, rbx, r9, rax = 0; + __m512i z0, z1, z2, z3, z4, z5, z6; + __mmask16 k1, k2, k3, k4; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)w1; + rdx = (word64)(word32)omega; + rcx = (word64)(size_t)h; + r8 = (word64)(size_t)idx; + + z3 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_make_hint_88_avx512_low, + 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_make_hint_88_avx512_neg_low, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_make_hint_88_avx512_seq, + 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_make_hint_88_avx512_step, + 0)); + z5 = _mm512_setzero_si512(); + rbx = (word64)(0); + rbx = (word32)((word32)WC_L8(r8, 0)); + rcx = (word64)(rcx + rbx); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 512)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 576)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 640)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 704)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 768)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 832)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 896)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 960)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_88_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + WC_S8(r8, 0) = (byte)((byte)rbx); + rax = (word32)(0); + goto L_mldsa_make_hint_88_avx512_done; +L_mldsa_make_hint_88_avx512_too_many: + rax = (word32)(-1); +L_mldsa_make_hint_88_avx512_done: + return (int)(word32)rax; +} + +XALIGNED(16) static const word32 L_mldsa_make_hint_32_avx512_low[] + WC_X64I_UNUSED = { + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, +}; + +XALIGNED(16) static const word32 L_mldsa_make_hint_32_avx512_neg_low[] + WC_X64I_UNUSED = { + 0xfffc0100, 0xfffc0100, 0xfffc0100, 0xfffc0100, + 0xfffc0100, 0xfffc0100, 0xfffc0100, 0xfffc0100, + 0xfffc0100, 0xfffc0100, 0xfffc0100, 0xfffc0100, + 0xfffc0100, 0xfffc0100, 0xfffc0100, 0xfffc0100, +}; + +XALIGNED(16) static const word32 L_mldsa_make_hint_32_avx512_seq[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000002, 0x00000003, + 0x00000004, 0x00000005, 0x00000006, 0x00000007, + 0x00000008, 0x00000009, 0x0000000a, 0x0000000b, + 0x0000000c, 0x0000000d, 0x0000000e, 0x0000000f, +}; + +XALIGNED(16) static const word32 L_mldsa_make_hint_32_avx512_step[] + WC_X64I_UNUSED = { + 0x00000010, 0x00000010, 0x00000010, 0x00000010, + 0x00000010, 0x00000010, 0x00000010, 0x00000010, + 0x00000010, 0x00000010, 0x00000010, 0x00000010, + 0x00000010, 0x00000010, 0x00000010, 0x00000010, +}; + +WC_X64I_TARGET("avx,avx512f,bmi,bmi2") +WOLFSSL_LOCAL int wc_mldsa_make_hint_32_avx512(const sword32* s, + const sword32* w1, unsigned int omega, byte* h, byte* idx) +{ + word64 rdi, rsi, rdx, rcx, r8, rbx, r9, rax = 0; + __m512i z0, z1, z2, z3, z4, z5, z6; + __mmask16 k1, k2, k3, k4; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)w1; + rdx = (word64)(word32)omega; + rcx = (word64)(size_t)h; + r8 = (word64)(size_t)idx; + + z3 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_make_hint_32_avx512_low, + 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_make_hint_32_avx512_neg_low, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_make_hint_32_avx512_seq, + 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_make_hint_32_avx512_step, + 0)); + z5 = _mm512_setzero_si512(); + rbx = (word64)(0); + rbx = (word32)((word32)WC_L8(r8, 0)); + rcx = (word64)(rcx + rbx); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 512)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 576)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 640)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 704)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 768)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 832)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 896)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + z1 = _mm512_add_epi32(z1, z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + k1 = _mm512_cmp_epi32_mask(z0, z3, 6); + k2 = _mm512_cmp_epi32_mask(z0, z4, 1); + k3 = _mm512_cmp_epi32_mask(z0, z4, 0); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 960)); + k4 = _mm512_cmp_epi32_mask(z6, z5, 4); + k3 = _kand_mask16(k3, k4); + k1 = _kor_mask16(k1, k2); + k1 = _kor_mask16(k1, k3); + z0 = _mm512_mask_compress_epi32(z0, k1, z1); + r9 = (word32)((word32)k1); + r9 = (word32)(WC_X64I_POPCNT32((word32)r9)); + rbx = (word32)((word32)rbx + (word32)r9); + if ((sword32)((word32)rbx) > (sword32)((word32)rdx)) { + goto L_mldsa_make_hint_32_avx512_too_many; + } + rax = (word32)(0xffffffff); + rax = (word32)((word32)_bzhi_u32((unsigned int)(word32)rax, (unsigned int)( + word32)r9)); + k4 = (__mmask16)(word32)rax; + _mm512_mask_cvtepi32_storeu_epi8(WC_PW(rcx, 0), k4, z0); + rcx = (word64)(rcx + r9); + WC_S8(r8, 0) = (byte)((byte)rbx); + rax = (word32)(0); + goto L_mldsa_make_hint_32_avx512_done; +L_mldsa_make_hint_32_avx512_too_many: + rax = (word32)(-1); +L_mldsa_make_hint_32_avx512_done: + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_use_hint_88_avx512(sword32* w1, const byte* h) +{ + word64 rdi, rsi, r12, rax, r8 = 0, rdx = 0, r9, r10, rcx, r11 = 0; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8; + __mmask16 k2, k3, k1; + + rdi = (word64)(size_t)w1; + rsi = (word64)(size_t)h; + + r12 = (word64)(0x7fe001); + z3 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r12)); + z3 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z3)); + r12 = (word64)(0x17400); + z4 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r12)); + z4 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z4)); + r12 = (word64)(0x2e800); + z5 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r12)); + z5 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z5)); + r12 = (word64)(0x3fefd4); + z6 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r12)); + z6 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z6)); + r12 = (word64)(0x2c); + z7 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r12)); + z7 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z7)); + r12 = (word64)(1); + z8 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r12)); + z8 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z8)); + rax = (word64)(0); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 0)) & 0xff); + /* 1/4 vectors */ + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 80)) & 0xff); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_0: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_0; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_0; +L_mldsa_use_hint_88_avx512_hints_done_0_0: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_1: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_1; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_1; +L_mldsa_use_hint_88_avx512_hints_done_0_1: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_2: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_2; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_2; +L_mldsa_use_hint_88_avx512_hints_done_0_2: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_3: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_3; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_3; +L_mldsa_use_hint_88_avx512_hints_done_0_3: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_4: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_4; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_4; +L_mldsa_use_hint_88_avx512_hints_done_0_4: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_5: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_5; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_5; +L_mldsa_use_hint_88_avx512_hints_done_0_5: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_6: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_6; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_6; +L_mldsa_use_hint_88_avx512_hints_done_0_6: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_7: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_7; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_7; +L_mldsa_use_hint_88_avx512_hints_done_0_7: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_8: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_8; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_8; +L_mldsa_use_hint_88_avx512_hints_done_0_8: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_9: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_9; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_9; +L_mldsa_use_hint_88_avx512_hints_done_0_9: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_10: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_10; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_10; +L_mldsa_use_hint_88_avx512_hints_done_0_10: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_11: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_11; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_11; +L_mldsa_use_hint_88_avx512_hints_done_0_11: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_12: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_12; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_12; +L_mldsa_use_hint_88_avx512_hints_done_0_12: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_13: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_13; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_13; +L_mldsa_use_hint_88_avx512_hints_done_0_13: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_14: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_14; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_14; +L_mldsa_use_hint_88_avx512_hints_done_0_14: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_0_15: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_0_15; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_0_15; +L_mldsa_use_hint_88_avx512_hints_done_0_15: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z2); + /* 2/4 vectors */ + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 81)) & 0xff); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1024)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_0: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_0; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_0; +L_mldsa_use_hint_88_avx512_hints_done_1_0: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1024), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1088)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_1: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_1; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_1; +L_mldsa_use_hint_88_avx512_hints_done_1_1: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1088), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1152)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_2: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_2; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_2; +L_mldsa_use_hint_88_avx512_hints_done_1_2: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1152), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1216)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_3: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_3; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_3; +L_mldsa_use_hint_88_avx512_hints_done_1_3: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1216), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1280)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_4: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_4; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_4; +L_mldsa_use_hint_88_avx512_hints_done_1_4: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1280), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1344)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_5: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_5; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_5; +L_mldsa_use_hint_88_avx512_hints_done_1_5: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1344), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1408)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_6: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_6; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_6; +L_mldsa_use_hint_88_avx512_hints_done_1_6: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1408), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1472)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_7: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_7; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_7; +L_mldsa_use_hint_88_avx512_hints_done_1_7: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1472), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1536)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_8: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_8; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_8; +L_mldsa_use_hint_88_avx512_hints_done_1_8: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1536), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1600)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_9: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_9; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_9; +L_mldsa_use_hint_88_avx512_hints_done_1_9: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1600), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1664)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_10: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_10; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_10; +L_mldsa_use_hint_88_avx512_hints_done_1_10: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1664), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1728)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_11: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_11; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_11; +L_mldsa_use_hint_88_avx512_hints_done_1_11: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1728), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1792)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_12: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_12; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_12; +L_mldsa_use_hint_88_avx512_hints_done_1_12: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1792), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1856)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_13: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_13; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_13; +L_mldsa_use_hint_88_avx512_hints_done_1_13: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1856), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1920)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_14: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_14; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_14; +L_mldsa_use_hint_88_avx512_hints_done_1_14: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1920), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1984)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_1_15: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_1_15; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_1_15; +L_mldsa_use_hint_88_avx512_hints_done_1_15: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 1984), z2); + /* 3/4 vectors */ + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 82)) & 0xff); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2048)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_0: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_0; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_0; +L_mldsa_use_hint_88_avx512_hints_done_2_0: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2048), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2112)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_1: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_1; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_1; +L_mldsa_use_hint_88_avx512_hints_done_2_1: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2112), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2176)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_2: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_2; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_2; +L_mldsa_use_hint_88_avx512_hints_done_2_2: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2176), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2240)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_3: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_3; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_3; +L_mldsa_use_hint_88_avx512_hints_done_2_3: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2240), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2304)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_4: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_4; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_4; +L_mldsa_use_hint_88_avx512_hints_done_2_4: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2304), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2368)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_5: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_5; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_5; +L_mldsa_use_hint_88_avx512_hints_done_2_5: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2368), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2432)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_6: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_6; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_6; +L_mldsa_use_hint_88_avx512_hints_done_2_6: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2432), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2496)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_7: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_7; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_7; +L_mldsa_use_hint_88_avx512_hints_done_2_7: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2496), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2560)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_8: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_8; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_8; +L_mldsa_use_hint_88_avx512_hints_done_2_8: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2560), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2624)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_9: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_9; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_9; +L_mldsa_use_hint_88_avx512_hints_done_2_9: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2624), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2688)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_10: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_10; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_10; +L_mldsa_use_hint_88_avx512_hints_done_2_10: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2688), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2752)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_11: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_11; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_11; +L_mldsa_use_hint_88_avx512_hints_done_2_11: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2752), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2816)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_12: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_12; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_12; +L_mldsa_use_hint_88_avx512_hints_done_2_12: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2816), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2880)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_13: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_13; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_13; +L_mldsa_use_hint_88_avx512_hints_done_2_13: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2880), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2944)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_14: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_14; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_14; +L_mldsa_use_hint_88_avx512_hints_done_2_14: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 2944), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3008)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_2_15: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_2_15; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_2_15; +L_mldsa_use_hint_88_avx512_hints_done_2_15: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3008), z2); + /* 4/4 vectors */ + rdx = (rdx & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, 83)) & 0xff); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3072)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_0: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_0; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_0; +L_mldsa_use_hint_88_avx512_hints_done_3_0: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3072), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3136)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_1: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_1; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_1; +L_mldsa_use_hint_88_avx512_hints_done_3_1: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3136), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3200)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_2: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_2; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_2; +L_mldsa_use_hint_88_avx512_hints_done_3_2: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3200), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3264)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_3: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_3; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_3; +L_mldsa_use_hint_88_avx512_hints_done_3_3: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3264), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3328)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_4: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_4; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_4; +L_mldsa_use_hint_88_avx512_hints_done_3_4: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3328), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3392)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_5: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_5; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_5; +L_mldsa_use_hint_88_avx512_hints_done_3_5: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3392), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3456)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_6: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_6; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_6; +L_mldsa_use_hint_88_avx512_hints_done_3_6: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3456), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3520)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_7: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_7; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_7; +L_mldsa_use_hint_88_avx512_hints_done_3_7: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3520), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3584)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_8: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_8; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_8; +L_mldsa_use_hint_88_avx512_hints_done_3_8: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3584), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3648)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_9: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_9; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_9; +L_mldsa_use_hint_88_avx512_hints_done_3_9: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3648), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3712)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_10: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_10; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_10; +L_mldsa_use_hint_88_avx512_hints_done_3_10: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3712), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3776)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_11: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_11; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_11; +L_mldsa_use_hint_88_avx512_hints_done_3_11: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3776), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3840)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_12: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_12; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_12; +L_mldsa_use_hint_88_avx512_hints_done_3_12: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3840), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3904)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_13: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_13; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_13; +L_mldsa_use_hint_88_avx512_hints_done_3_13: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3904), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3968)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_14: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_14; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_14; +L_mldsa_use_hint_88_avx512_hints_done_3_14: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 3968), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 4032)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_mullo_epi32(z0, z7); + z2 = _mm512_add_epi32(z2, z6); + z2 = _mm512_srli_epi32(z2, 23); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + k2 = _mm512_cmp_epi32_mask(z2, z7, _MM_CMPINT_EQ); + z1 = _mm512_mask_blend_epi32(k2, z1, _mm512_sub_epi32(z1, z8)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + r9 = (word64)(1); + r10 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_88_avx512_hints_next_3_15: + if ((sword8)((byte)rax) >= (sword8)((byte)rdx)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r8) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_88_avx512_hints_done_3_15; + } + r11 = (word64)(r9); + r11 = (word64)(r11 << ((byte)rcx & 63)); + r10 = (word64)(r10 | r11); + rax = (rax & ~(word64)0xff) | ((word64)(byte)((byte)rax + 1) & 0xff); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rsi, rax)) & 0xff); + goto L_mldsa_use_hint_88_avx512_hints_next_3_15; +L_mldsa_use_hint_88_avx512_hints_done_3_15: + k1 = (__mmask16)(word32)r10; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + k3 = _mm512_cmp_epi32_mask(z7, z2, _MM_CMPINT_NLE); + z2 = _mm512_maskz_mov_epi32(k3, z2); + z0 = _mm512_srai_epi32(z2, 31); + z0 = _mm512_and_si512(z0, z7); + z2 = _mm512_add_epi32(z2, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 4032), z2); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_use_hint_32_avx512(sword32* w1, byte k, + const byte* h) +{ + word64 rdi, rsi, rdx, r14, r8, r9 = 0, r13, rax = 0, r10 = 0, r11 = 0, + rcx = 0, r12 = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3, z4, z5, z6, z7, z8; + __mmask16 k1; + + rdi = (word64)(size_t)w1; + rsi = (word64)(word64)k; + rdx = (word64)(size_t)h; + + r14 = (word64)(0x7fe001); + z3 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z3 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z3)); + r14 = (word64)(0x3ff00); + z4 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z4 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z4)); + r14 = (word64)(0x7fe00); + z5 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z5 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z5)); + r14 = (word64)(0x3feff); + z6 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z6 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z6)); + r14 = (word64)(0xf); + z7 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z7 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z7)); + r14 = (word64)(1); + z8 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z8 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z8)); + r8 = (word64)(0); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, 0)) & 0xff); + r13 = (word64)(rsi * 10); + r13 = (word64)(r13 - 5); +L_mldsa_use_hint_32_avx512_start_256: + rax = (rax & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r13)) & 0xff); + r13 = (word64)(r13 + 1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_0: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_0; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_0; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_0; +L_mldsa_use_hint_32_avx512_hints_done_0: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_1: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_1; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x10) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_1; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_1; +L_mldsa_use_hint_32_avx512_hints_done_1: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_2: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_2; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x20) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_2; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_2; +L_mldsa_use_hint_32_avx512_hints_done_2: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_3: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_3; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x30) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_3; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_3; +L_mldsa_use_hint_32_avx512_hints_done_3: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_4: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_4; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x40) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_4; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_4; +L_mldsa_use_hint_32_avx512_hints_done_4: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_5: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_5; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x50) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_5; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_5; +L_mldsa_use_hint_32_avx512_hints_done_5: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_6: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_6; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x60) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_6; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_6; +L_mldsa_use_hint_32_avx512_hints_done_6: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_7: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_7; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x70) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_7; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_7; +L_mldsa_use_hint_32_avx512_hints_done_7: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_8: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_8; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x80) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_8; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_8; +L_mldsa_use_hint_32_avx512_hints_done_8: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_9: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_9; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0x90) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_9; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_9; +L_mldsa_use_hint_32_avx512_hints_done_9: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_10: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_10; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xa0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_10; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_10; +L_mldsa_use_hint_32_avx512_hints_done_10: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_11: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_11; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xb0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_11; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_11; +L_mldsa_use_hint_32_avx512_hints_done_11: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_12: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_12; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xc0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_12; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_12; +L_mldsa_use_hint_32_avx512_hints_done_12: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_13: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_13; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xd0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_13; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_13; +L_mldsa_use_hint_32_avx512_hints_done_13: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_14: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_14; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xe0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_14; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_14; +L_mldsa_use_hint_32_avx512_hints_done_14: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z2); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z1 = _mm512_srai_epi32(z0, 31); + z1 = _mm512_and_si512(z1, z3); + z0 = _mm512_add_epi32(z0, z1); + z2 = _mm512_add_epi32(z0, z6); + z2 = _mm512_srli_epi32(z2, 19); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z4, z1); + z1 = _mm512_srli_epi32(z1, 31); + z2 = _mm512_add_epi32(z2, z1); + z1 = _mm512_mullo_epi32(z2, z5); + z1 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_srli_epi32(z2, 4); + z1 = _mm512_sub_epi32(z1, z0); + z2 = _mm512_and_si512(z2, z7); + r10 = (word64)(1); + r11 = (word64)(0); + rcx = (word64)(0); +L_mldsa_use_hint_32_avx512_hints_next_15: + if ((sword8)((byte)r8) >= (sword8)((byte)rax)) { + goto L_mldsa_use_hint_32_avx512_hints_done_15; + } + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)r9) & 0xff); + rcx = (rcx & ~(word64)0xff) | ((word64)(byte)((byte)rcx - 0xf0) & 0xff); + if ((sword64)(rcx) >= (sword64)(0x10)) { + goto L_mldsa_use_hint_32_avx512_hints_done_15; + } + r12 = (word64)(r10); + r12 = (word64)(r12 << ((byte)rcx & 63)); + r11 = (word64)(r11 | r12); + r8 = (r8 & ~(word64)0xff) | ((word64)(byte)((byte)r8 + 1) & 0xff); + r9 = (r9 & ~(word64)0xff) | ((word64)(byte)(WC_L8(rdx, r8)) & 0xff); + goto L_mldsa_use_hint_32_avx512_hints_next_15; +L_mldsa_use_hint_32_avx512_hints_done_15: + k1 = (__mmask16)(word32)r11; + z1 = _mm512_sub_epi32(z1, z8); + z1 = _mm512_srli_epi32(z1, 31); + z1 = _mm512_slli_epi32(z1, 1); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_add_epi32(z2, z1)); + z2 = _mm512_and_si512(z2, z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z2); + rdi = (word64)(rdi + 0x400); + rsi = (word64)(rsi - 1); + if ((rsi) != (0)) { + goto L_mldsa_use_hint_32_avx512_start_256; + } +} + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void wc_mldsa_redistribute_21_rand_x8_avx512(const word64* s, + byte* out, word32 stride) +{ + word64 rdi, rsi, rdx, r13, rax, rcx, r8, r9, r10, r11, r12, r14; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)out; + rdx = (word64)(word32)stride; + + r13 = (word32)((word32)rdx); + rax = (word64)(rsi); + rax = (word64)(rax + r13); + rcx = (word64)(rax); + rcx = (word64)(rcx + r13); + r8 = (word64)(rcx); + r8 = (word64)(r8 + r13); + r9 = (word64)(r8); + r9 = (word64)(r9 + r13); + r10 = (word64)(r9); + r10 = (word64)(r10 + r13); + r11 = (word64)(r10); + r11 = (word64)(r11 + r13); + r12 = (word64)(r11); + r12 = (word64)(r12 + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 0), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 0), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 0), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 0), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z7); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 64), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 64), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 64), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 64), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z7); + r14 = (word64)(WC_L64(rdi, 1024)); + WC_S64(rsi, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1032)); + WC_S64(rax, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1040)); + WC_S64(rcx, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1048)); + WC_S64(r8, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1056)); + WC_S64(r9, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1064)); + WC_S64(r10, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1072)); + WC_S64(r11, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1080)); + WC_S64(r12, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1088)); + WC_S64(rsi, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1096)); + WC_S64(rax, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1104)); + WC_S64(rcx, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1112)); + WC_S64(r8, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1120)); + WC_S64(r9, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1128)); + WC_S64(r10, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1136)); + WC_S64(r11, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1144)); + WC_S64(r12, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1152)); + WC_S64(rsi, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1160)); + WC_S64(rax, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1168)); + WC_S64(rcx, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1176)); + WC_S64(r8, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1184)); + WC_S64(r9, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1192)); + WC_S64(r10, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1200)); + WC_S64(r11, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1208)); + WC_S64(r12, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1216)); + WC_S64(rsi, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1224)); + WC_S64(rax, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1232)); + WC_S64(rcx, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1240)); + WC_S64(r8, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1248)); + WC_S64(r9, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1256)); + WC_S64(r10, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1264)); + WC_S64(r11, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1272)); + WC_S64(r12, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1280)); + WC_S64(rsi, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1288)); + WC_S64(rax, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1296)); + WC_S64(rcx, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1304)); + WC_S64(r8, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1312)); + WC_S64(r9, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1320)); + WC_S64(r10, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1328)); + WC_S64(r11, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1336)); + WC_S64(r12, 160) = (word64)(r14); +} + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void wc_mldsa_redistribute_17_rand_x8_avx512(const word64* s, + byte* out, word32 stride) +{ + word64 rdi, rsi, rdx, r13, rax, rcx, r8, r9, r10, r11, r12, r14; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)out; + rdx = (word64)(word32)stride; + + r13 = (word32)((word32)rdx); + rax = (word64)(rsi); + rax = (word64)(rax + r13); + rcx = (word64)(rax); + rcx = (word64)(rcx + r13); + r8 = (word64)(rcx); + r8 = (word64)(r8 + r13); + r9 = (word64)(r8); + r9 = (word64)(r9 + r13); + r10 = (word64)(r9); + r10 = (word64)(r10 + r13); + r11 = (word64)(r10); + r11 = (word64)(r11 + r13); + r12 = (word64)(r11); + r12 = (word64)(r12 + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 0), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 0), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 0), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 0), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z7); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 64), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 64), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 64), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 64), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z7); + r14 = (word64)(WC_L64(rdi, 1024)); + WC_S64(rsi, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1032)); + WC_S64(rax, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1040)); + WC_S64(rcx, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1048)); + WC_S64(r8, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1056)); + WC_S64(r9, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1064)); + WC_S64(r10, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1072)); + WC_S64(r11, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1080)); + WC_S64(r12, 128) = (word64)(r14); +} + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_17_x2_avx512_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff, + 0x04, 0x05, 0x06, 0xff, 0x06, 0x07, 0x08, 0xff, + 0xff, 0x01, 0x02, 0x03, 0xff, 0x03, 0x04, 0x05, + 0xff, 0x05, 0x06, 0x07, 0xff, 0x07, 0x08, 0x09, +}; + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_17_x2_avx512_shuff_1[] + WC_X64I_UNUSED = { + 0x02, 0x03, 0x04, 0xff, 0x04, 0x05, 0x06, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0xff, 0x03, 0x04, 0x05, 0xff, 0x05, 0x06, 0x07, + 0xff, 0x07, 0x08, 0x09, 0xff, 0x09, 0x0a, 0x0b, +}; + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_17_x2_avx512_shuff_2[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0x06, 0xff, 0x06, 0x07, 0x08, 0xff, + 0x08, 0x09, 0x0a, 0xff, 0x0a, 0x0b, 0x0c, 0xff, + 0xff, 0x05, 0x06, 0x07, 0xff, 0x07, 0x08, 0x09, + 0xff, 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0x0d, +}; + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_17_x2_avx512_shuff_3[] + WC_X64I_UNUSED = { + 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0x0a, 0x0b, 0x0c, 0xff, 0x0c, 0x0d, 0x0e, 0xff, + 0xff, 0x07, 0x08, 0x09, 0xff, 0x09, 0x0a, 0x0b, + 0xff, 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_17_x2_avx512_vs_8[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000002, 0x00000004, 0x00000006, + 0x00000008, 0x0000000a, 0x0000000c, 0x0000000e, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_17_x2_avx512_mask[] + WC_X64I_UNUSED = { + 0x0003ffff, 0x0003ffff, 0x0003ffff, 0x0003ffff, + 0x0003ffff, 0x0003ffff, 0x0003ffff, 0x0003ffff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_17_x2_avx512_gamma17[] + WC_X64I_UNUSED = { + 0x00020000, 0x00020000, 0x00020000, 0x00020000, + 0x00020000, 0x00020000, 0x00020000, 0x00020000, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void wc_mldsa_decode_gamma1_17_x2_avx512(const byte* sA, + const byte* sB, sword32* zA, sword32* zB) +{ + word64 rdi, rsi, rdx, rcx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14; + + rdi = (word64)(size_t)sA; + rsi = (word64)(size_t)sB; + rdx = (word64)(size_t)zA; + rcx = (word64)(size_t)zB; + + z14 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm21, 0)); + z7 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_x2_avx512_shuff_0, 0))); + z8 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_x2_avx512_shuff_1, 0))); + z9 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_x2_avx512_shuff_2, 0))); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_x2_avx512_shuff_3, 0))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_x2_avx512_vs_8, 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_x2_avx512_mask, 0))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_17_x2_avx512_gamma17, 0))); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 1); + /* 0/15 */ + z6 = _mm512_permutex_epi64(z0, 0x94); + z6 = _mm512_shuffle_epi8(z6, z7); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_extracti64x4_epi64(z6, + 1)); + /* 1/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z0, z6, z1); + z6 = _mm512_permutex_epi64(z6, 0x94); + z6 = _mm512_shuffle_epi8(z6, z8); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_extracti64x4_epi64(z6, + 1)); + /* 2/15 */ + z6 = _mm512_permutex_epi64(z1, 0x94); + z6 = _mm512_shuffle_epi8(z6, z9); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), _mm512_extracti64x4_epi64(z6, + 1)); + /* 3/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z1, z6, z2); + z6 = _mm512_permutex_epi64(z6, 0x94); + z6 = _mm512_shuffle_epi8(z6, z10); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), _mm512_extracti64x4_epi64(z6, + 1)); + /* 4/15 */ + z6 = _mm512_permutex_epi64(z2, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z7); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), _mm512_extracti64x4_epi64(z6, + 1)); + /* 5/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z2, z6, z3); + z6 = _mm512_permutex_epi64(z6, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z8); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 160), _mm512_extracti64x4_epi64(z6, + 1)); + /* 6/15 */ + z6 = _mm512_permutex_epi64(z3, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z9); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 192), _mm512_extracti64x4_epi64(z6, + 1)); + /* 7/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z3, z6, z4); + z6 = _mm512_permutex_epi64(z6, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z10); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 224), _mm512_extracti64x4_epi64(z6, + 1)); + /* 8/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z4, z6, z5); + z6 = _mm512_permutex_epi64(z6, 0x94); + z6 = _mm512_shuffle_epi8(z6, z7); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 256), _mm512_extracti64x4_epi64(z6, + 1)); + /* 9/15 */ + z6 = _mm512_permutex_epi64(z5, 0x94); + z6 = _mm512_shuffle_epi8(z6, z8); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 288), _mm512_extracti64x4_epi64(z6, + 1)); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256)), 1); + /* 10/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z5, z6, z0); + z6 = _mm512_permutex_epi64(z6, 0x94); + z6 = _mm512_shuffle_epi8(z6, z9); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 320), _mm512_extracti64x4_epi64(z6, + 1)); + /* 11/15 */ + z6 = _mm512_permutex_epi64(z0, 0x94); + z6 = _mm512_shuffle_epi8(z6, z10); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 352), _mm512_extracti64x4_epi64(z6, + 1)); + /* 12/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z0, z6, z1); + z6 = _mm512_permutex_epi64(z6, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z7); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 384), _mm512_extracti64x4_epi64(z6, + 1)); + /* 13/15 */ + z6 = _mm512_permutex_epi64(z1, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z8); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 416), _mm512_extracti64x4_epi64(z6, + 1)); + /* 14/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z1, z6, z2); + z6 = _mm512_permutex_epi64(z6, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z9); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 448), _mm512_extracti64x4_epi64(z6, + 1)); + /* 15/15 */ + z6 = _mm512_permutex_epi64(z2, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z10); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 480), _mm512_extracti64x4_epi64(z6, + 1)); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + /* 0/15 */ + z6 = _mm512_permutex_epi64(z0, 0x94); + z6 = _mm512_shuffle_epi8(z6, z7); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 512), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 512), _mm512_extracti64x4_epi64(z6, + 1)); + /* 1/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z0, z6, z1); + z6 = _mm512_permutex_epi64(z6, 0x94); + z6 = _mm512_shuffle_epi8(z6, z8); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 544), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 544), _mm512_extracti64x4_epi64(z6, + 1)); + /* 2/15 */ + z6 = _mm512_permutex_epi64(z1, 0x94); + z6 = _mm512_shuffle_epi8(z6, z9); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 576), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 576), _mm512_extracti64x4_epi64(z6, + 1)); + /* 3/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z1, z6, z2); + z6 = _mm512_permutex_epi64(z6, 0x94); + z6 = _mm512_shuffle_epi8(z6, z10); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 608), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 608), _mm512_extracti64x4_epi64(z6, + 1)); + /* 4/15 */ + z6 = _mm512_permutex_epi64(z2, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z7); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 640), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 640), _mm512_extracti64x4_epi64(z6, + 1)); + /* 5/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z2, z6, z3); + z6 = _mm512_permutex_epi64(z6, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z8); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 672), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 672), _mm512_extracti64x4_epi64(z6, + 1)); + /* 6/15 */ + z6 = _mm512_permutex_epi64(z3, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z9); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 704), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 704), _mm512_extracti64x4_epi64(z6, + 1)); + /* 7/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z3, z6, z4); + z6 = _mm512_permutex_epi64(z6, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z10); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 736), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 736), _mm512_extracti64x4_epi64(z6, + 1)); + /* 8/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z4, z6, z5); + z6 = _mm512_permutex_epi64(z6, 0x94); + z6 = _mm512_shuffle_epi8(z6, z7); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 768), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 768), _mm512_extracti64x4_epi64(z6, + 1)); + /* 9/15 */ + z6 = _mm512_permutex_epi64(z5, 0x94); + z6 = _mm512_shuffle_epi8(z6, z8); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 800), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 800), _mm512_extracti64x4_epi64(z6, + 1)); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 512))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 512)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 544))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 544)), 1); + /* 10/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z5, z6, z0); + z6 = _mm512_permutex_epi64(z6, 0x94); + z6 = _mm512_shuffle_epi8(z6, z9); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 832), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 832), _mm512_extracti64x4_epi64(z6, + 1)); + /* 11/15 */ + z6 = _mm512_permutex_epi64(z0, 0x94); + z6 = _mm512_shuffle_epi8(z6, z10); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 864), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 864), _mm512_extracti64x4_epi64(z6, + 1)); + /* 12/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z0, z6, z1); + z6 = _mm512_permutex_epi64(z6, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z7); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 896), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 896), _mm512_extracti64x4_epi64(z6, + 1)); + /* 13/15 */ + z6 = _mm512_permutex_epi64(z1, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z8); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 928), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 928), _mm512_extracti64x4_epi64(z6, + 1)); + /* 14/15 */ + z6 = z14; + z6 = _mm512_permutex2var_epi64(z1, z6, z2); + z6 = _mm512_permutex_epi64(z6, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z9); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 960), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 960), _mm512_extracti64x4_epi64(z6, + 1)); + /* 15/15 */ + z6 = _mm512_permutex_epi64(z2, 0xe9); + z6 = _mm512_shuffle_epi8(z6, z10); + z6 = _mm512_srlv_epi32(z6, z11); + z6 = _mm512_and_si512(z6, z12); + z6 = _mm512_sub_epi32(z13, z6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 992), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 992), _mm512_extracti64x4_epi64(z6, + 1)); +} + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_19_x2_avx512_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff, + 0x05, 0x06, 0x07, 0xff, 0x07, 0x08, 0x09, 0xff, + 0xff, 0x02, 0x03, 0x04, 0xff, 0x04, 0x05, 0x06, + 0xff, 0x07, 0x08, 0x09, 0xff, 0x09, 0x0a, 0x0b, +}; + +XALIGNED(16) static const byte L_mldsa_decode_gamma1_19_x2_avx512_shuff_1[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0x06, 0xff, 0x06, 0x07, 0x08, 0xff, + 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0x0d, 0xff, + 0xff, 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0x0a, + 0xff, 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_19_x2_avx512_vs_8[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00000008, 0x0000000c, 0x00000008, 0x0000000c, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_19_x2_avx512_mask[] + WC_X64I_UNUSED = { + 0x000fffff, 0x000fffff, 0x000fffff, 0x000fffff, + 0x000fffff, 0x000fffff, 0x000fffff, 0x000fffff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_gamma1_19_x2_avx512_gamma19[] + WC_X64I_UNUSED = { + 0x00080000, 0x00080000, 0x00080000, 0x00080000, + 0x00080000, 0x00080000, 0x00080000, 0x00080000, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void wc_mldsa_decode_gamma1_19_x2_avx512(const byte* sA, + const byte* sB, sword32* zA, sword32* zB) +{ + word64 rdi, rsi, rdx, rcx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11; + + rdi = (word64)(size_t)sA; + rsi = (word64)(size_t)sB; + rdx = (word64)(size_t)zA; + rcx = (word64)(size_t)zB; + + z11 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm21, 0)); + z6 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_19_x2_avx512_shuff_0, 0))); + z7 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_19_x2_avx512_shuff_1, 0))); + z8 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_19_x2_avx512_vs_8, 0))); + z9 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_19_x2_avx512_mask, 0))); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_gamma1_19_x2_avx512_gamma19, 0))); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128)), 1); + /* 0/7 */ + z5 = _mm512_permutex_epi64(z0, 0x94); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_extracti64x4_epi64(z5, + 1)); + /* 1/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z0, z5, z1); + z5 = _mm512_permutex_epi64(z5, 0x94); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_extracti64x4_epi64(z5, + 1)); + /* 2/7 */ + z5 = _mm512_permutex_epi64(z1, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), _mm512_extracti64x4_epi64(z5, + 1)); + /* 3/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z1, z5, z2); + z5 = _mm512_permutex_epi64(z5, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), _mm512_extracti64x4_epi64(z5, + 1)); + /* 4/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z2, z5, z3); + z5 = _mm512_permutex_epi64(z5, 0x94); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), _mm512_extracti64x4_epi64(z5, + 1)); + /* 5/7 */ + z5 = _mm512_permutex_epi64(z3, 0x94); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 160), _mm512_extracti64x4_epi64(z5, + 1)); + /* 6/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z3, z5, z4); + z5 = _mm512_permutex_epi64(z5, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 192), _mm512_extracti64x4_epi64(z5, + 1)); + /* 7/7 */ + z5 = _mm512_permutex_epi64(z4, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 224), _mm512_extracti64x4_epi64(z5, + 1)); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288)), 1); + /* 0/7 */ + z5 = _mm512_permutex_epi64(z0, 0x94); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 256), _mm512_extracti64x4_epi64(z5, + 1)); + /* 1/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z0, z5, z1); + z5 = _mm512_permutex_epi64(z5, 0x94); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 288), _mm512_extracti64x4_epi64(z5, + 1)); + /* 2/7 */ + z5 = _mm512_permutex_epi64(z1, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 320), _mm512_extracti64x4_epi64(z5, + 1)); + /* 3/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z1, z5, z2); + z5 = _mm512_permutex_epi64(z5, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 352), _mm512_extracti64x4_epi64(z5, + 1)); + /* 4/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z2, z5, z3); + z5 = _mm512_permutex_epi64(z5, 0x94); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 384), _mm512_extracti64x4_epi64(z5, + 1)); + /* 5/7 */ + z5 = _mm512_permutex_epi64(z3, 0x94); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 416), _mm512_extracti64x4_epi64(z5, + 1)); + /* 6/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z3, z5, z4); + z5 = _mm512_permutex_epi64(z5, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 448), _mm512_extracti64x4_epi64(z5, + 1)); + /* 7/7 */ + z5 = _mm512_permutex_epi64(z4, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 480), _mm512_extracti64x4_epi64(z5, + 1)); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + /* 0/7 */ + z5 = _mm512_permutex_epi64(z0, 0x94); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 512), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 512), _mm512_extracti64x4_epi64(z5, + 1)); + /* 1/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z0, z5, z1); + z5 = _mm512_permutex_epi64(z5, 0x94); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 544), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 544), _mm512_extracti64x4_epi64(z5, + 1)); + /* 2/7 */ + z5 = _mm512_permutex_epi64(z1, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 576), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 576), _mm512_extracti64x4_epi64(z5, + 1)); + /* 3/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z1, z5, z2); + z5 = _mm512_permutex_epi64(z5, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 608), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 608), _mm512_extracti64x4_epi64(z5, + 1)); + /* 4/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z2, z5, z3); + z5 = _mm512_permutex_epi64(z5, 0x94); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 640), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 640), _mm512_extracti64x4_epi64(z5, + 1)); + /* 5/7 */ + z5 = _mm512_permutex_epi64(z3, 0x94); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 672), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 672), _mm512_extracti64x4_epi64(z5, + 1)); + /* 6/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z3, z5, z4); + z5 = _mm512_permutex_epi64(z5, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 704), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 704), _mm512_extracti64x4_epi64(z5, + 1)); + /* 7/7 */ + z5 = _mm512_permutex_epi64(z4, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 736), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 736), _mm512_extracti64x4_epi64(z5, + 1)); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 512))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 512)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 544))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 544)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 576))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 576)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 608))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 608)), 1); + /* 0/7 */ + z5 = _mm512_permutex_epi64(z0, 0x94); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 768), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 768), _mm512_extracti64x4_epi64(z5, + 1)); + /* 1/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z0, z5, z1); + z5 = _mm512_permutex_epi64(z5, 0x94); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 800), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 800), _mm512_extracti64x4_epi64(z5, + 1)); + /* 2/7 */ + z5 = _mm512_permutex_epi64(z1, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 832), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 832), _mm512_extracti64x4_epi64(z5, + 1)); + /* 3/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z1, z5, z2); + z5 = _mm512_permutex_epi64(z5, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 864), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 864), _mm512_extracti64x4_epi64(z5, + 1)); + /* 4/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z2, z5, z3); + z5 = _mm512_permutex_epi64(z5, 0x94); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 896), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 896), _mm512_extracti64x4_epi64(z5, + 1)); + /* 5/7 */ + z5 = _mm512_permutex_epi64(z3, 0x94); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 928), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 928), _mm512_extracti64x4_epi64(z5, + 1)); + /* 6/7 */ + z5 = z11; + z5 = _mm512_permutex2var_epi64(z3, z5, z4); + z5 = _mm512_permutex_epi64(z5, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z6); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 960), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 960), _mm512_extracti64x4_epi64(z5, + 1)); + /* 7/7 */ + z5 = _mm512_permutex_epi64(z4, 0xe9); + z5 = _mm512_shuffle_epi8(z5, z7); + z5 = _mm512_srlv_epi32(z5, z8); + z5 = _mm512_and_si512(z5, z9); + z5 = _mm512_sub_epi32(z10, z5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 992), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 992), _mm512_extracti64x4_epi64(z5, + 1)); +} + +XALIGNED(16) static const byte L_mldsa_decode_t0_x2_avx512_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0x03, + 0x03, 0x04, 0x05, 0xff, 0x04, 0x05, 0x06, 0x07, + 0x06, 0x07, 0x08, 0xff, 0xff, 0x08, 0x09, 0xff, + 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0x0b, 0x0c, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_x2_avx512_shuff_1[] + WC_X64I_UNUSED = { + 0x05, 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, 0x08, + 0x08, 0x09, 0xff, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x03, 0x04, 0x05, 0xff, 0xff, 0x05, 0x06, 0xff, + 0x06, 0x07, 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_x2_avx512_shuff_2[] + WC_X64I_UNUSED = { + 0x02, 0x03, 0xff, 0xff, 0xff, 0x03, 0x04, 0x05, + 0x05, 0x06, 0xff, 0xff, 0x06, 0x07, 0x08, 0xff, + 0x00, 0x01, 0x02, 0xff, 0xff, 0x02, 0x03, 0xff, + 0x03, 0x04, 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_x2_avx512_shuff_3[] + WC_X64I_UNUSED = { + 0x07, 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, + 0x0a, 0x0b, 0xff, 0xff, 0x0b, 0x0c, 0x0d, 0xff, + 0x05, 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0xff, + 0x08, 0x09, 0x0a, 0xff, 0xff, 0xff, 0x0a, 0x0b, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_x2_avx512_shuff_4[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0x07, + 0x07, 0x08, 0xff, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0x02, 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0xff, + 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_x2_avx512_shuff_5[] + WC_X64I_UNUSED = { + 0x01, 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0x04, + 0x04, 0x05, 0xff, 0xff, 0x05, 0x06, 0x07, 0xff, + 0x07, 0x08, 0x09, 0xff, 0xff, 0x09, 0x0a, 0xff, + 0x0a, 0x0b, 0x0c, 0xff, 0xff, 0xff, 0x0c, 0x0d, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_x2_avx512_shuff_6[] + WC_X64I_UNUSED = { + 0x06, 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, 0x09, + 0x09, 0x0a, 0xff, 0xff, 0x0a, 0x0b, 0x0c, 0xff, + 0x04, 0x05, 0x06, 0xff, 0xff, 0x06, 0x07, 0x08, + 0x07, 0x08, 0x09, 0xff, 0xff, 0xff, 0x09, 0x0a, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t0_x2_avx512_shuff_7[] + WC_X64I_UNUSED = { + 0x03, 0x04, 0xff, 0xff, 0xff, 0x04, 0x05, 0x06, + 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0x09, 0xff, + 0x01, 0x02, 0x03, 0xff, 0xff, 0x03, 0x04, 0xff, + 0x04, 0x05, 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t0_x2_avx512_vs_8[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000d, 0x00000002, 0x00000007, + 0x00000004, 0x00000009, 0x00000006, 0x00000013, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t0_x2_avx512_mask[] + WC_X64I_UNUSED = { + 0x00001fff, 0x00001fff, 0x00001fff, 0x00001fff, + 0x00001fff, 0x00001fff, 0x00001fff, 0x00001fff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t0_x2_avx512_d_max_half[] + WC_X64I_UNUSED = { + 0x00001000, 0x00001000, 0x00001000, 0x00001000, + 0x00001000, 0x00001000, 0x00001000, 0x00001000, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void wc_mldsa_decode_t0_x2_avx512(const byte* t0A, + const byte* t0B, sword32* tA, sword32* tB) +{ + word64 rdi, rsi, rdx, rcx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17; + + rdi = (word64)(size_t)t0A; + rsi = (word64)(size_t)t0B; + rdx = (word64)(size_t)tA; + rcx = (word64)(size_t)tB; + + z16 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm21, 0)); + z5 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_shuff_0, 0))); + z6 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_shuff_1, 0))); + z7 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_shuff_2, 0))); + z8 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_shuff_3, 0))); + z9 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_shuff_4, 0))); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_shuff_5, 0))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_shuff_6, 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_shuff_7, 0))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_vs_8, 0))); + z14 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_mask, 0))); + z15 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t0_x2_avx512_d_max_half, 0))); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + /* 1/32 */ + z4 = _mm512_permutex_epi64(z0, 0x44); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_extracti64x4_epi64(z4, + 1)); + /* 2/32 */ + z4 = _mm512_permutex_epi64(z0, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_extracti64x4_epi64(z4, + 1)); + /* 3/32 */ + z17 = z16; + z17 = _mm512_permutex2var_epi64(z0, z17, z1); + z0 = z17; + z4 = _mm512_permutex_epi64(z0, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), _mm512_extracti64x4_epi64(z4, + 1)); + /* 4/32 */ + z4 = _mm512_permutex_epi64(z1, 0x94); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), _mm512_extracti64x4_epi64(z4, + 1)); + /* 5/32 */ + z17 = z16; + z17 = _mm512_permutex2var_epi64(z1, z17, z2); + z1 = z17; + z4 = _mm512_permutex_epi64(z1, 0x94); + z4 = _mm512_shuffle_epi8(z4, z9); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), _mm512_extracti64x4_epi64(z4, + 1)); + /* 6/32 */ + z4 = _mm512_permutex_epi64(z2, 0x44); + z4 = _mm512_shuffle_epi8(z4, z10); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 160), _mm512_extracti64x4_epi64(z4, + 1)); + /* 7/32 */ + z4 = _mm512_permutex_epi64(z2, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z11); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 192), _mm512_extracti64x4_epi64(z4, + 1)); + /* 8/32 */ + z17 = z16; + z17 = _mm512_permutex2var_epi64(z2, z17, z3); + z2 = z17; + z4 = _mm512_permutex_epi64(z2, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z12); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 224), _mm512_extracti64x4_epi64(z4, + 1)); + /* 9/32 */ + z4 = _mm512_permutex_epi64(z3, 0x99); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 256), _mm512_extracti64x4_epi64(z4, + 1)); + /* 10/32 */ + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 1); + z17 = z16; + z17 = _mm512_permutex2var_epi64(z3, z17, z0); + z3 = z17; + z4 = _mm512_permutex_epi64(z3, 0x94); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 288), _mm512_extracti64x4_epi64(z4, + 1)); + /* 11/32 */ + z4 = _mm512_permutex_epi64(z0, 0x94); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 320), _mm512_extracti64x4_epi64(z4, + 1)); + /* 12/32 */ + z4 = _mm512_permutex_epi64(z0, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 352), _mm512_extracti64x4_epi64(z4, + 1)); + /* 13/32 */ + z17 = z16; + z17 = _mm512_permutex2var_epi64(z0, z17, z1); + z0 = z17; + z4 = _mm512_permutex_epi64(z0, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z9); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 384), _mm512_extracti64x4_epi64(z4, + 1)); + /* 14/32 */ + z4 = _mm512_permutex_epi64(z1, 0x99); + z4 = _mm512_shuffle_epi8(z4, z10); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 416), _mm512_extracti64x4_epi64(z4, + 1)); + /* 15/32 */ + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z17 = z16; + z17 = _mm512_permutex2var_epi64(z1, z17, z2); + z1 = z17; + z4 = _mm512_permutex_epi64(z1, 0x94); + z4 = _mm512_shuffle_epi8(z4, z11); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 448), _mm512_extracti64x4_epi64(z4, + 1)); + /* 16/32 */ + z4 = _mm512_permutex_epi64(z2, 0x94); + z4 = _mm512_shuffle_epi8(z4, z12); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 480), _mm512_extracti64x4_epi64(z4, + 1)); + /* 17/32 */ + z4 = _mm512_permutex_epi64(z2, 0xee); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 512), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 512), _mm512_extracti64x4_epi64(z4, + 1)); + /* 18/32 */ + z17 = z16; + z17 = _mm512_permutex2var_epi64(z2, z17, z3); + z2 = z17; + z4 = _mm512_permutex_epi64(z2, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 544), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 544), _mm512_extracti64x4_epi64(z4, + 1)); + /* 19/32 */ + z4 = _mm512_permutex_epi64(z3, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 576), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 576), _mm512_extracti64x4_epi64(z4, + 1)); + /* 20/32 */ + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288)), 1); + z17 = z16; + z17 = _mm512_permutex2var_epi64(z3, z17, z0); + z3 = z17; + z4 = _mm512_permutex_epi64(z3, 0x94); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 608), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 608), _mm512_extracti64x4_epi64(z4, + 1)); + /* 21/32 */ + z4 = _mm512_permutex_epi64(z0, 0x94); + z4 = _mm512_shuffle_epi8(z4, z9); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 640), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 640), _mm512_extracti64x4_epi64(z4, + 1)); + /* 22/32 */ + z4 = _mm512_permutex_epi64(z0, 0xee); + z4 = _mm512_shuffle_epi8(z4, z10); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 672), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 672), _mm512_extracti64x4_epi64(z4, + 1)); + /* 23/32 */ + z17 = z16; + z17 = _mm512_permutex2var_epi64(z0, z17, z1); + z0 = z17; + z4 = _mm512_permutex_epi64(z0, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z11); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 704), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 704), _mm512_extracti64x4_epi64(z4, + 1)); + /* 24/32 */ + z4 = _mm512_permutex_epi64(z1, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z12); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 736), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 736), _mm512_extracti64x4_epi64(z4, + 1)); + /* 25/32 */ + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z17 = z16; + z17 = _mm512_permutex2var_epi64(z1, z17, z2); + z1 = z17; + z4 = _mm512_permutex_epi64(z1, 0x99); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 768), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 768), _mm512_extracti64x4_epi64(z4, + 1)); + /* 26/32 */ + z4 = _mm512_permutex_epi64(z2, 0x94); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 800), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 800), _mm512_extracti64x4_epi64(z4, + 1)); + /* 27/32 */ + z4 = _mm512_permutex_epi64(z2, 0x3e); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 832), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 832), _mm512_extracti64x4_epi64(z4, + 1)); + /* 28/32 */ + z17 = z16; + z17 = _mm512_permutex2var_epi64(z2, z17, z3); + z2 = z17; + z4 = _mm512_permutex_epi64(z2, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 864), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 864), _mm512_extracti64x4_epi64(z4, + 1)); + /* 29/32 */ + z4 = _mm512_permutex_epi64(z3, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z9); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 896), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 896), _mm512_extracti64x4_epi64(z4, + 1)); + /* 30/32 */ + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384)), 1); + z17 = z16; + z17 = _mm512_permutex2var_epi64(z3, z17, z0); + z3 = z17; + z4 = _mm512_permutex_epi64(z3, 0x99); + z4 = _mm512_shuffle_epi8(z4, z10); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 928), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 928), _mm512_extracti64x4_epi64(z4, + 1)); + /* 31/32 */ + z4 = _mm512_permutex_epi64(z0, 0x94); + z4 = _mm512_shuffle_epi8(z4, z11); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 960), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 960), _mm512_extracti64x4_epi64(z4, + 1)); + /* 32/32 */ + z4 = _mm512_permutex_epi64(z0, 0x3e); + z4 = _mm512_shuffle_epi8(z4, z12); + z4 = _mm512_srlv_epi32(z4, z13); + z4 = _mm512_and_si512(z4, z14); + z4 = _mm512_sub_epi32(z15, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 992), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 992), _mm512_extracti64x4_epi64(z4, + 1)); +} + +XALIGNED(16) static const byte L_mldsa_decode_t1_x2_avx512_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0xff, + 0x02, 0x03, 0xff, 0xff, 0xff, 0x03, 0x04, 0xff, + 0x05, 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, 0xff, + 0x07, 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t1_x2_avx512_shuff_1[] + WC_X64I_UNUSED = { + 0x02, 0x03, 0xff, 0xff, 0xff, 0x03, 0x04, 0xff, + 0x04, 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, + 0x07, 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, + 0x09, 0x0a, 0x08, 0xff, 0xff, 0x0a, 0x0b, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t1_x2_avx512_shuff_2[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, + 0x06, 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, 0xff, + 0x01, 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0xff, + 0x03, 0x04, 0xff, 0xff, 0xff, 0x04, 0x05, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_t1_x2_avx512_shuff_3[] + WC_X64I_UNUSED = { + 0x06, 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, 0xff, + 0x08, 0x09, 0xff, 0xff, 0xff, 0x09, 0x0a, 0xff, + 0x03, 0x04, 0xff, 0xff, 0xff, 0x04, 0x05, 0xff, + 0x05, 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t1_x2_avx512_vs_8[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000a, 0x00000004, 0x0000000e, + 0x00000000, 0x0000000a, 0x00000004, 0x0000000e, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_t1_x2_avx512_mask[] + WC_X64I_UNUSED = { + 0x000003ff, 0x000003ff, 0x000003ff, 0x000003ff, + 0x000003ff, 0x000003ff, 0x000003ff, 0x000003ff, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void wc_mldsa_decode_t1_x2_avx512(const byte* t1A, + const byte* t1B, sword32* tA, sword32* tB) +{ + word64 rdi, rsi, rdx, rcx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12; + + rdi = (word64)(size_t)t1A; + rsi = (word64)(size_t)t1B; + rdx = (word64)(size_t)tA; + rcx = (word64)(size_t)tB; + + z11 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm21, 0)); + z5 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_x2_avx512_shuff_0, 0))); + z6 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_x2_avx512_shuff_1, 0))); + z7 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_x2_avx512_shuff_2, 0))); + z8 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_x2_avx512_shuff_3, 0))); + z9 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_x2_avx512_vs_8, 0))); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_t1_x2_avx512_mask, 0))); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + /* 1/32 */ + z4 = _mm512_permutex_epi64(z0, 0x44); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_extracti64x4_epi64(z4, + 1)); + /* 2/32 */ + z4 = _mm512_permutex_epi64(z0, 0x99); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_extracti64x4_epi64(z4, + 1)); + /* 3/32 */ + z4 = _mm512_permutex_epi64(z0, 0x3e); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), _mm512_extracti64x4_epi64(z4, + 1)); + /* 4/32 */ + z12 = z11; + z12 = _mm512_permutex2var_epi64(z0, z12, z1); + z0 = z12; + z4 = _mm512_permutex_epi64(z0, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), _mm512_extracti64x4_epi64(z4, + 1)); + /* 5/32 */ + z4 = _mm512_permutex_epi64(z1, 0x99); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), _mm512_extracti64x4_epi64(z4, + 1)); + /* 6/32 */ + z4 = _mm512_permutex_epi64(z1, 0xee); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 160), _mm512_extracti64x4_epi64(z4, + 1)); + /* 7/32 */ + z12 = z11; + z12 = _mm512_permutex2var_epi64(z1, z12, z2); + z1 = z12; + z4 = _mm512_permutex_epi64(z1, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 192), _mm512_extracti64x4_epi64(z4, + 1)); + /* 8/32 */ + z4 = _mm512_permutex_epi64(z2, 0x94); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 224), _mm512_extracti64x4_epi64(z4, + 1)); + /* 9/32 */ + z4 = _mm512_permutex_epi64(z2, 0xee); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 256), _mm512_extracti64x4_epi64(z4, + 1)); + /* 10/32 */ + z12 = z11; + z12 = _mm512_permutex2var_epi64(z2, z12, z3); + z2 = z12; + z4 = _mm512_permutex_epi64(z2, 0x99); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 288), _mm512_extracti64x4_epi64(z4, + 1)); + /* 11/32 */ + z4 = _mm512_permutex_epi64(z3, 0x94); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 320), _mm512_extracti64x4_epi64(z4, + 1)); + /* 12/32 */ + z4 = _mm512_permutex_epi64(z3, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 352), _mm512_extracti64x4_epi64(z4, + 1)); + /* 13/32 */ + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 1); + z12 = z11; + z12 = _mm512_permutex2var_epi64(z3, z12, z0); + z3 = z12; + z4 = _mm512_permutex_epi64(z3, 0x99); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 384), _mm512_extracti64x4_epi64(z4, + 1)); + /* 14/32 */ + z4 = _mm512_permutex_epi64(z0, 0x44); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 416), _mm512_extracti64x4_epi64(z4, + 1)); + /* 15/32 */ + z4 = _mm512_permutex_epi64(z0, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 448), _mm512_extracti64x4_epi64(z4, + 1)); + /* 16/32 */ + z4 = _mm512_permutex_epi64(z0, 0x3e); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 480), _mm512_extracti64x4_epi64(z4, + 1)); + /* 17/32 */ + z4 = _mm512_permutex_epi64(z1, 0x44); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 512), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 512), _mm512_extracti64x4_epi64(z4, + 1)); + /* 18/32 */ + z4 = _mm512_permutex_epi64(z1, 0x99); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 544), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 544), _mm512_extracti64x4_epi64(z4, + 1)); + /* 19/32 */ + z4 = _mm512_permutex_epi64(z1, 0x3e); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 576), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 576), _mm512_extracti64x4_epi64(z4, + 1)); + /* 20/32 */ + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z12 = z11; + z12 = _mm512_permutex2var_epi64(z1, z12, z2); + z1 = z12; + z4 = _mm512_permutex_epi64(z1, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 608), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 608), _mm512_extracti64x4_epi64(z4, + 1)); + /* 21/32 */ + z4 = _mm512_permutex_epi64(z2, 0x99); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 640), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 640), _mm512_extracti64x4_epi64(z4, + 1)); + /* 22/32 */ + z4 = _mm512_permutex_epi64(z2, 0xee); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 672), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 672), _mm512_extracti64x4_epi64(z4, + 1)); + /* 23/32 */ + z12 = z11; + z12 = _mm512_permutex2var_epi64(z2, z12, z3); + z2 = z12; + z4 = _mm512_permutex_epi64(z2, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 704), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 704), _mm512_extracti64x4_epi64(z4, + 1)); + /* 24/32 */ + z4 = _mm512_permutex_epi64(z3, 0x94); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 736), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 736), _mm512_extracti64x4_epi64(z4, + 1)); + /* 25/32 */ + z4 = _mm512_permutex_epi64(z3, 0xee); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 768), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 768), _mm512_extracti64x4_epi64(z4, + 1)); + /* 26/32 */ + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288)), 1); + z12 = z11; + z12 = _mm512_permutex2var_epi64(z3, z12, z0); + z3 = z12; + z4 = _mm512_permutex_epi64(z3, 0x99); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 800), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 800), _mm512_extracti64x4_epi64(z4, + 1)); + /* 27/32 */ + z4 = _mm512_permutex_epi64(z0, 0x94); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 832), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 832), _mm512_extracti64x4_epi64(z4, + 1)); + /* 28/32 */ + z4 = _mm512_permutex_epi64(z0, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 864), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 864), _mm512_extracti64x4_epi64(z4, + 1)); + /* 29/32 */ + z12 = z11; + z12 = _mm512_permutex2var_epi64(z0, z12, z1); + z0 = z12; + z4 = _mm512_permutex_epi64(z0, 0x99); + z4 = _mm512_shuffle_epi8(z4, z5); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 896), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 896), _mm512_extracti64x4_epi64(z4, + 1)); + /* 30/32 */ + z4 = _mm512_permutex_epi64(z1, 0x44); + z4 = _mm512_shuffle_epi8(z4, z6); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 928), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 928), _mm512_extracti64x4_epi64(z4, + 1)); + /* 31/32 */ + z4 = _mm512_permutex_epi64(z1, 0xe9); + z4 = _mm512_shuffle_epi8(z4, z7); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 960), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 960), _mm512_extracti64x4_epi64(z4, + 1)); + /* 32/32 */ + z4 = _mm512_permutex_epi64(z1, 0x3e); + z4 = _mm512_shuffle_epi8(z4, z8); + z4 = _mm512_srlv_epi32(z4, z9); + z4 = _mm512_and_si512(z4, z10); + z4 = _mm512_slli_epi32(z4, 13); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 992), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 992), _mm512_extracti64x4_epi64(z4, + 1)); +} + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_x2_avx512_shuff_0[] + WC_X64I_UNUSED = { + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, + 0x00, 0x01, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_x2_avx512_shuff_1[] + WC_X64I_UNUSED = { + 0x01, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0x02, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_x2_avx512_shuff_2[] + WC_X64I_UNUSED = { + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x02, 0x03, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0x04, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_x2_avx512_shuff_3[] + WC_X64I_UNUSED = { + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x03, 0x04, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0x05, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_x2_avx512_shuff_4[] + WC_X64I_UNUSED = { + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x04, 0x05, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_x2_avx512_shuff_5[] + WC_X64I_UNUSED = { + 0x05, 0xff, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x05, 0x06, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0x07, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_x2_avx512_shuff_6[] + WC_X64I_UNUSED = { + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x06, 0x07, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0x08, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_decode_eta_2_x2_avx512_shuff_7[] + WC_X64I_UNUSED = { + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x07, 0x08, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, 0xff, + 0x09, 0xff, 0xff, 0xff, 0x09, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_2_x2_avx512_two[] + WC_X64I_UNUSED = { + 0x00000002, 0x00000002, 0x00000002, 0x00000002, + 0x00000002, 0x00000002, 0x00000002, 0x00000002, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_2_x2_avx512_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000003, 0x00000006, 0x00000001, + 0x00000004, 0x00000007, 0x00000002, 0x00000005, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_2_x2_avx512_mask[] + WC_X64I_UNUSED = { + 0x00000007, 0x00000007, 0x00000007, 0x00000007, + 0x00000007, 0x00000007, 0x00000007, 0x00000007, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void wc_mldsa_decode_eta_2_x2_avx512(const byte* pA, + const byte* pB, sword32* sA, sword32* sB) +{ + word64 rdi, rsi, rdx, rcx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16; + + rdi = (word64)(size_t)pA; + rsi = (word64)(size_t)pB; + rdx = (word64)(size_t)sA; + rcx = (word64)(size_t)sB; + + z15 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm20, 0)); + z4 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_shuff_0, 0))); + z5 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_shuff_1, 0))); + z6 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_shuff_2, 0))); + z7 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_shuff_3, 0))); + z8 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_shuff_4, 0))); + z9 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_shuff_5, 0))); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_shuff_6, 0))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_shuff_7, 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_two, 0))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_vs, 0))); + z14 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_2_x2_avx512_mask, 0))); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z4); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z7); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z10); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), _mm512_extracti64x4_epi64(z3, + 1)); + z0 = _mm512_permutex_epi64(z0, 0x39); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z5); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z8); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z11); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 160), _mm512_extracti64x4_epi64(z3, + 1)); + z0 = _mm512_permutex_epi64(z0, 0x39); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z6); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 192), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z9); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 224), _mm512_extracti64x4_epi64(z3, + 1)); + z0 = _mm512_permutex_epi64(z0, 0x39); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z4); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 256), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z7); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 288), _mm512_extracti64x4_epi64(z3, + 1)); + z16 = z15; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z0 = z16; + z0 = _mm512_permutex_epi64(z0, 0x38); + z3 = _mm512_permutex_epi64(z0, 0x44); + z3 = _mm512_shuffle_epi8(z3, z10); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 320), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z5); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 352), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z8); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 384), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z11); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 416), _mm512_extracti64x4_epi64(z3, + 1)); + z1 = _mm512_permutex_epi64(z1, 0x39); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z6); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 448), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z9); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 480), _mm512_extracti64x4_epi64(z3, + 1)); + z1 = _mm512_permutex_epi64(z1, 0x39); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z4); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 512), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 512), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z7); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 544), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 544), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z10); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 576), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 576), _mm512_extracti64x4_epi64(z3, + 1)); + z1 = _mm512_permutex_epi64(z1, 0x39); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z5); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 608), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 608), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z8); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 640), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 640), _mm512_extracti64x4_epi64(z3, + 1)); + z16 = z15; + z16 = _mm512_permutex2var_epi64(z1, z16, z2); + z1 = z16; + z1 = _mm512_permutex_epi64(z1, 0x38); + z3 = _mm512_permutex_epi64(z1, 0x44); + z3 = _mm512_shuffle_epi8(z3, z11); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 672), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 672), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z6); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 704), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 704), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z9); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 736), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 736), _mm512_extracti64x4_epi64(z3, + 1)); + z2 = _mm512_permutex_epi64(z2, 0x39); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z4); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 768), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 768), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z7); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 800), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 800), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z10); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 832), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 832), _mm512_extracti64x4_epi64(z3, + 1)); + z2 = _mm512_permutex_epi64(z2, 0x39); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z5); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 864), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 864), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z8); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 896), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 896), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z11); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 928), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 928), _mm512_extracti64x4_epi64(z3, + 1)); + z2 = _mm512_permutex_epi64(z2, 0x39); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z6); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 960), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 960), _mm512_extracti64x4_epi64(z3, + 1)); + z3 = _mm512_permutex_epi64(z2, 0x44); + z3 = _mm512_shuffle_epi8(z3, z9); + z3 = _mm512_srlv_epi32(z3, z13); + z3 = _mm512_and_si512(z3, z14); + z3 = _mm512_sub_epi32(z12, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 992), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 992), _mm512_extracti64x4_epi64(z3, + 1)); +} + +XALIGNED(16) static const word32 L_mldsa_decode_eta_4_x2_avx512_four[] + WC_X64I_UNUSED = { + 0x00000004, 0x00000004, 0x00000004, 0x00000004, + 0x00000004, 0x00000004, 0x00000004, 0x00000004, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_4_x2_avx512_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000008, 0x0000000c, + 0x00000010, 0x00000014, 0x00000018, 0x0000001c, +}; + +XALIGNED(16) static const word32 L_mldsa_decode_eta_4_x2_avx512_mask[] + WC_X64I_UNUSED = { + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, +}; + +WC_X64I_TARGET("avx512f,avx512vl") +WOLFSSL_LOCAL void wc_mldsa_decode_eta_4_x2_avx512(const byte* pA, + const byte* pB, sword32* sA, sword32* sB) +{ + word64 rdi, rsi, rdx, rcx, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10; + __mmask16 k1; + + rdi = (word64)(size_t)pA; + rsi = (word64)(size_t)pB; + rdx = (word64)(size_t)sA; + rcx = (word64)(size_t)sB; + + rax = (word32)(0xff00); + k1 = (__mmask16)(word32)rax; + z8 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_4_x2_avx512_four, 0))); + z9 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_4_x2_avx512_vs, 0))); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_decode_eta_4_x2_avx512_mask, 0))); + z0 = _mm512_set1_epi32((int)WC_L32(rdi, 0)); + z0 = _mm512_mask_blend_epi32(k1, z0, _mm512_set1_epi32((int)WC_L32(rsi, + 0))); + z1 = _mm512_set1_epi32((int)WC_L32(rdi, 4)); + z1 = _mm512_mask_blend_epi32(k1, z1, _mm512_set1_epi32((int)WC_L32(rsi, + 4))); + z2 = _mm512_set1_epi32((int)WC_L32(rdi, 8)); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_set1_epi32((int)WC_L32(rsi, + 8))); + z3 = _mm512_set1_epi32((int)WC_L32(rdi, 12)); + z3 = _mm512_mask_blend_epi32(k1, z3, _mm512_set1_epi32((int)WC_L32(rsi, + 12))); + z4 = _mm512_set1_epi32((int)WC_L32(rdi, 16)); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_set1_epi32((int)WC_L32(rsi, + 16))); + z5 = _mm512_set1_epi32((int)WC_L32(rdi, 20)); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_set1_epi32((int)WC_L32(rsi, + 20))); + z6 = _mm512_set1_epi32((int)WC_L32(rdi, 24)); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_set1_epi32((int)WC_L32(rsi, + 24))); + z7 = _mm512_set1_epi32((int)WC_L32(rdi, 28)); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_set1_epi32((int)WC_L32(rsi, + 28))); + z0 = _mm512_srlv_epi32(z0, z9); + z1 = _mm512_srlv_epi32(z1, z9); + z2 = _mm512_srlv_epi32(z2, z9); + z3 = _mm512_srlv_epi32(z3, z9); + z4 = _mm512_srlv_epi32(z4, z9); + z5 = _mm512_srlv_epi32(z5, z9); + z6 = _mm512_srlv_epi32(z6, z9); + z7 = _mm512_srlv_epi32(z7, z9); + z0 = _mm512_and_si512(z0, z10); + z1 = _mm512_and_si512(z1, z10); + z2 = _mm512_and_si512(z2, z10); + z3 = _mm512_and_si512(z3, z10); + z4 = _mm512_and_si512(z4, z10); + z5 = _mm512_and_si512(z5, z10); + z6 = _mm512_and_si512(z6, z10); + z7 = _mm512_and_si512(z7, z10); + z0 = _mm512_sub_epi32(z8, z0); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_sub_epi32(z8, z2); + z3 = _mm512_sub_epi32(z8, z3); + z4 = _mm512_sub_epi32(z8, z4); + z5 = _mm512_sub_epi32(z8, z5); + z6 = _mm512_sub_epi32(z8, z6); + z7 = _mm512_sub_epi32(z8, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 160), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 192), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 224), _mm512_extracti64x4_epi64(z7, + 1)); + z0 = _mm512_set1_epi32((int)WC_L32(rdi, 32)); + z0 = _mm512_mask_blend_epi32(k1, z0, _mm512_set1_epi32((int)WC_L32(rsi, + 32))); + z1 = _mm512_set1_epi32((int)WC_L32(rdi, 36)); + z1 = _mm512_mask_blend_epi32(k1, z1, _mm512_set1_epi32((int)WC_L32(rsi, + 36))); + z2 = _mm512_set1_epi32((int)WC_L32(rdi, 40)); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_set1_epi32((int)WC_L32(rsi, + 40))); + z3 = _mm512_set1_epi32((int)WC_L32(rdi, 44)); + z3 = _mm512_mask_blend_epi32(k1, z3, _mm512_set1_epi32((int)WC_L32(rsi, + 44))); + z4 = _mm512_set1_epi32((int)WC_L32(rdi, 48)); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_set1_epi32((int)WC_L32(rsi, + 48))); + z5 = _mm512_set1_epi32((int)WC_L32(rdi, 52)); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_set1_epi32((int)WC_L32(rsi, + 52))); + z6 = _mm512_set1_epi32((int)WC_L32(rdi, 56)); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_set1_epi32((int)WC_L32(rsi, + 56))); + z7 = _mm512_set1_epi32((int)WC_L32(rdi, 60)); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_set1_epi32((int)WC_L32(rsi, + 60))); + z0 = _mm512_srlv_epi32(z0, z9); + z1 = _mm512_srlv_epi32(z1, z9); + z2 = _mm512_srlv_epi32(z2, z9); + z3 = _mm512_srlv_epi32(z3, z9); + z4 = _mm512_srlv_epi32(z4, z9); + z5 = _mm512_srlv_epi32(z5, z9); + z6 = _mm512_srlv_epi32(z6, z9); + z7 = _mm512_srlv_epi32(z7, z9); + z0 = _mm512_and_si512(z0, z10); + z1 = _mm512_and_si512(z1, z10); + z2 = _mm512_and_si512(z2, z10); + z3 = _mm512_and_si512(z3, z10); + z4 = _mm512_and_si512(z4, z10); + z5 = _mm512_and_si512(z5, z10); + z6 = _mm512_and_si512(z6, z10); + z7 = _mm512_and_si512(z7, z10); + z0 = _mm512_sub_epi32(z8, z0); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_sub_epi32(z8, z2); + z3 = _mm512_sub_epi32(z8, z3); + z4 = _mm512_sub_epi32(z8, z4); + z5 = _mm512_sub_epi32(z8, z5); + z6 = _mm512_sub_epi32(z8, z6); + z7 = _mm512_sub_epi32(z8, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 256), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 288), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 320), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 352), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 384), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 416), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 448), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 480), _mm512_extracti64x4_epi64(z7, + 1)); + z0 = _mm512_set1_epi32((int)WC_L32(rdi, 64)); + z0 = _mm512_mask_blend_epi32(k1, z0, _mm512_set1_epi32((int)WC_L32(rsi, + 64))); + z1 = _mm512_set1_epi32((int)WC_L32(rdi, 68)); + z1 = _mm512_mask_blend_epi32(k1, z1, _mm512_set1_epi32((int)WC_L32(rsi, + 68))); + z2 = _mm512_set1_epi32((int)WC_L32(rdi, 72)); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_set1_epi32((int)WC_L32(rsi, + 72))); + z3 = _mm512_set1_epi32((int)WC_L32(rdi, 76)); + z3 = _mm512_mask_blend_epi32(k1, z3, _mm512_set1_epi32((int)WC_L32(rsi, + 76))); + z4 = _mm512_set1_epi32((int)WC_L32(rdi, 80)); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_set1_epi32((int)WC_L32(rsi, + 80))); + z5 = _mm512_set1_epi32((int)WC_L32(rdi, 84)); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_set1_epi32((int)WC_L32(rsi, + 84))); + z6 = _mm512_set1_epi32((int)WC_L32(rdi, 88)); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_set1_epi32((int)WC_L32(rsi, + 88))); + z7 = _mm512_set1_epi32((int)WC_L32(rdi, 92)); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_set1_epi32((int)WC_L32(rsi, + 92))); + z0 = _mm512_srlv_epi32(z0, z9); + z1 = _mm512_srlv_epi32(z1, z9); + z2 = _mm512_srlv_epi32(z2, z9); + z3 = _mm512_srlv_epi32(z3, z9); + z4 = _mm512_srlv_epi32(z4, z9); + z5 = _mm512_srlv_epi32(z5, z9); + z6 = _mm512_srlv_epi32(z6, z9); + z7 = _mm512_srlv_epi32(z7, z9); + z0 = _mm512_and_si512(z0, z10); + z1 = _mm512_and_si512(z1, z10); + z2 = _mm512_and_si512(z2, z10); + z3 = _mm512_and_si512(z3, z10); + z4 = _mm512_and_si512(z4, z10); + z5 = _mm512_and_si512(z5, z10); + z6 = _mm512_and_si512(z6, z10); + z7 = _mm512_and_si512(z7, z10); + z0 = _mm512_sub_epi32(z8, z0); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_sub_epi32(z8, z2); + z3 = _mm512_sub_epi32(z8, z3); + z4 = _mm512_sub_epi32(z8, z4); + z5 = _mm512_sub_epi32(z8, z5); + z6 = _mm512_sub_epi32(z8, z6); + z7 = _mm512_sub_epi32(z8, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 512), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 512), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 544), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 544), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 576), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 576), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 608), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 608), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 640), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 640), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 672), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 672), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 704), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 704), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 736), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 736), _mm512_extracti64x4_epi64(z7, + 1)); + z0 = _mm512_set1_epi32((int)WC_L32(rdi, 96)); + z0 = _mm512_mask_blend_epi32(k1, z0, _mm512_set1_epi32((int)WC_L32(rsi, + 96))); + z1 = _mm512_set1_epi32((int)WC_L32(rdi, 100)); + z1 = _mm512_mask_blend_epi32(k1, z1, _mm512_set1_epi32((int)WC_L32(rsi, + 100))); + z2 = _mm512_set1_epi32((int)WC_L32(rdi, 104)); + z2 = _mm512_mask_blend_epi32(k1, z2, _mm512_set1_epi32((int)WC_L32(rsi, + 104))); + z3 = _mm512_set1_epi32((int)WC_L32(rdi, 108)); + z3 = _mm512_mask_blend_epi32(k1, z3, _mm512_set1_epi32((int)WC_L32(rsi, + 108))); + z4 = _mm512_set1_epi32((int)WC_L32(rdi, 112)); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_set1_epi32((int)WC_L32(rsi, + 112))); + z5 = _mm512_set1_epi32((int)WC_L32(rdi, 116)); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_set1_epi32((int)WC_L32(rsi, + 116))); + z6 = _mm512_set1_epi32((int)WC_L32(rdi, 120)); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_set1_epi32((int)WC_L32(rsi, + 120))); + z7 = _mm512_set1_epi32((int)WC_L32(rdi, 124)); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_set1_epi32((int)WC_L32(rsi, + 124))); + z0 = _mm512_srlv_epi32(z0, z9); + z1 = _mm512_srlv_epi32(z1, z9); + z2 = _mm512_srlv_epi32(z2, z9); + z3 = _mm512_srlv_epi32(z3, z9); + z4 = _mm512_srlv_epi32(z4, z9); + z5 = _mm512_srlv_epi32(z5, z9); + z6 = _mm512_srlv_epi32(z6, z9); + z7 = _mm512_srlv_epi32(z7, z9); + z0 = _mm512_and_si512(z0, z10); + z1 = _mm512_and_si512(z1, z10); + z2 = _mm512_and_si512(z2, z10); + z3 = _mm512_and_si512(z3, z10); + z4 = _mm512_and_si512(z4, z10); + z5 = _mm512_and_si512(z5, z10); + z6 = _mm512_and_si512(z6, z10); + z7 = _mm512_and_si512(z7, z10); + z0 = _mm512_sub_epi32(z8, z0); + z1 = _mm512_sub_epi32(z8, z1); + z2 = _mm512_sub_epi32(z8, z2); + z3 = _mm512_sub_epi32(z8, z3); + z4 = _mm512_sub_epi32(z8, z4); + z5 = _mm512_sub_epi32(z8, z5); + z6 = _mm512_sub_epi32(z8, z6); + z7 = _mm512_sub_epi32(z8, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 768), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 768), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 800), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 800), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 832), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 832), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 864), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 864), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 896), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 896), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 928), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 928), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 960), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 960), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 992), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 992), _mm512_extracti64x4_epi64(z7, + 1)); +} + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_17_avx512_gamma[] + WC_X64I_UNUSED = { + 0x00020000, 0x00020000, 0x00020000, 0x00020000, + 0x00020000, 0x00020000, 0x00020000, 0x00020000, + 0x00020000, 0x00020000, 0x00020000, 0x00020000, + 0x00020000, 0x00020000, 0x00020000, 0x00020000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_17_avx512_perm0[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000003, 0x00000005, + 0x00000007, 0x00000008, 0x0000000a, 0x0000000c, + 0x0000000e, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_17_avx512_shift0[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000e, 0x0000000a, 0x00000006, + 0x00000002, 0x00000010, 0x0000000c, 0x00000008, + 0x00000004, 0x00000020, 0x00000020, 0x00000020, + 0x00000020, 0x00000020, 0x00000020, 0x00000020, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_17_avx512_perm1[] + WC_X64I_UNUSED = { + 0x00000001, 0x00000002, 0x00000004, 0x00000006, + 0x00000008, 0x00000009, 0x0000000b, 0x0000000d, + 0x0000000f, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_17_avx512_shift1[] + WC_X64I_UNUSED = { + 0x00000012, 0x00000004, 0x00000008, 0x0000000c, + 0x00000010, 0x00000002, 0x00000006, 0x0000000a, + 0x0000000e, 0x00000020, 0x00000020, 0x00000020, + 0x00000020, 0x00000020, 0x00000020, 0x00000020, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_17_avx512_perm2[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000003, 0x00000005, 0x00000007, + 0x00000000, 0x0000000a, 0x0000000c, 0x0000000e, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_17_avx512_shift2[] + WC_X64I_UNUSED = { + 0x00000020, 0x00000016, 0x0000001a, 0x0000001e, + 0x00000020, 0x00000014, 0x00000018, 0x0000001c, + 0x00000020, 0x00000020, 0x00000020, 0x00000020, + 0x00000020, 0x00000020, 0x00000020, 0x00000020, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_encode_gamma1_17_avx512(const sword32* z, byte* s) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10; + __mmask16 k1; + + rdi = (word64)(size_t)z; + rsi = (word64)(size_t)s; + + z10 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_17_avx512_gamma, 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_17_avx512_perm0, 0)); + z7 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_17_avx512_shift0, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_17_avx512_perm1, 0)); + z8 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_17_avx512_shift1, 0)); + z6 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_17_avx512_perm2, 0)); + z9 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_17_avx512_shift2, 0)); + rdx = (word32)(0x1ff); + k1 = (__mmask16)(word32)rdx; + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 36), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 72), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 108), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 144), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 180), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 216), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 252), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 288), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 324), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 360), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 396), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 432), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 468), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 504), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_mask_storeu_epi32(WC_PW(rsi, 540), k1, z1); +} + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_19_avx512_gamma[] + WC_X64I_UNUSED = { + 0x00080000, 0x00080000, 0x00080000, 0x00080000, + 0x00080000, 0x00080000, 0x00080000, 0x00080000, + 0x00080000, 0x00080000, 0x00080000, 0x00080000, + 0x00080000, 0x00080000, 0x00080000, 0x00080000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_19_avx512_perm0[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000003, 0x00000004, + 0x00000006, 0x00000008, 0x00000009, 0x0000000b, + 0x0000000c, 0x0000000e, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_19_avx512_shift0[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000c, 0x00000004, 0x00000010, + 0x00000008, 0x00000000, 0x0000000c, 0x00000004, + 0x00000010, 0x00000008, 0x00000020, 0x00000020, + 0x00000020, 0x00000020, 0x00000020, 0x00000020, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_19_avx512_perm1[] + WC_X64I_UNUSED = { + 0x00000001, 0x00000002, 0x00000004, 0x00000005, + 0x00000007, 0x00000009, 0x0000000a, 0x0000000c, + 0x0000000d, 0x0000000f, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_19_avx512_shift1[] + WC_X64I_UNUSED = { + 0x00000014, 0x00000008, 0x00000010, 0x00000004, + 0x0000000c, 0x00000014, 0x00000008, 0x00000010, + 0x00000004, 0x0000000c, 0x00000020, 0x00000020, + 0x00000020, 0x00000020, 0x00000020, 0x00000020, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_19_avx512_perm2[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000003, 0x00000000, 0x00000006, + 0x00000000, 0x00000000, 0x0000000b, 0x00000000, + 0x0000000e, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_gamma1_19_avx512_shift2[] + WC_X64I_UNUSED = { + 0x00000020, 0x0000001c, 0x00000020, 0x00000018, + 0x00000020, 0x00000020, 0x0000001c, 0x00000020, + 0x00000018, 0x00000020, 0x00000020, 0x00000020, + 0x00000020, 0x00000020, 0x00000020, 0x00000020, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_encode_gamma1_19_avx512(const sword32* z, byte* s) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10; + __mmask16 k1; + + rdi = (word64)(size_t)z; + rsi = (word64)(size_t)s; + + z10 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_19_avx512_gamma, 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_19_avx512_perm0, 0)); + z7 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_19_avx512_shift0, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_19_avx512_perm1, 0)); + z8 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_19_avx512_shift1, 0)); + z6 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_19_avx512_perm2, 0)); + z9 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_gamma1_19_avx512_shift2, 0)); + rdx = (word32)(0x3ff); + k1 = (__mmask16)(word32)rdx; + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 40), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 80), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 120), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 160), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 200), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 240), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 280), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 320), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 360), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 400), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 440), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 480), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 520), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 560), z1); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z0 = _mm512_sub_epi32(z10, z0); + z1 = _mm512_permutexvar_epi32(z4, z0); + z2 = _mm512_permutexvar_epi32(z5, z0); + z3 = _mm512_permutexvar_epi32(z6, z0); + z1 = _mm512_srlv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z9); + z1 = _mm512_or_si512(z1, z2); + z1 = _mm512_or_si512(z1, z3); + _mm512_mask_storeu_epi32(WC_PW(rsi, 600), k1, z1); +} + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_x2_avx512_shuff_0_even[] + WC_X64I_UNUSED = { + 0x00, 0x09, 0x0a, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x00, 0x09, 0x0a, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_x2_avx512_shuff_0_odd[] + WC_X64I_UNUSED = { + 0x04, 0x05, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x04, 0x05, 0x0e, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_x2_avx512_shuff_1_even[] + WC_X64I_UNUSED = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x09, + 0x0a, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x00, 0x09, 0x0a, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_x2_avx512_shuff_1_odd[] + WC_X64I_UNUSED = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x04, 0x05, + 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x04, 0x05, 0x0e, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_w1_88_x2_avx512_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000006, 0x0000000c, 0x00000012, + 0x00000000, 0x00000006, 0x0000000c, 0x00000012, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void wc_mldsa_encode_w1_88_x2_avx512(const sword32* w1A, + const sword32* w1B, byte* w1eA, byte* w1eB) +{ + word64 rdi, rsi, rdx, rcx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8; + + rdi = (word64)(size_t)w1A; + rsi = (word64)(size_t)w1B; + rdx = (word64)(size_t)w1eA; + rcx = (word64)(size_t)w1eB; + + z4 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_88_x2_avx512_shuff_0_even, 0))); + z5 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_88_x2_avx512_shuff_0_odd, 0))); + z6 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_88_x2_avx512_shuff_1_even, 0))); + z7 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_88_x2_avx512_shuff_1_odd, 0))); + z8 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_w1_88_x2_avx512_vs, 0))); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 512))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 512)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 544))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 544)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 576))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 576)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 608))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 608)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 640))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 640)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 672))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 672)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 704))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 704)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 736))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 736)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 768))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 768)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 800))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 800)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 832))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 832)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 864))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 864)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 896))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 896)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 928))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 928)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 960))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 960)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 992))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 992)), 1); + z0 = _mm512_sllv_epi32(z0, z8); + z1 = _mm512_sllv_epi32(z1, z8); + z2 = _mm512_shuffle_epi8(z0, z5); + z0 = _mm512_shuffle_epi8(z0, z4); + z3 = _mm512_shuffle_epi8(z1, z7); + z1 = _mm512_shuffle_epi8(z1, z6); + z0 = _mm512_or_si512(z0, z2); + z0 = _mm512_or_si512(z0, z3); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z2); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + z2 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z2)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z2), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); +} + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void wc_mldsa_encode_w1_32_avx512(const sword32* w1, byte* w1e) +{ + word64 rdi, rsi; + __m512i z0, z1; + + rdi = (word64)(size_t)w1; + rsi = (word64)(size_t)w1e; + + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 0), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 8), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 16), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 24), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 32), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 40), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 48), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 56), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 64), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 72), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 80), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 88), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 96), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 104), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 112), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 120), (__mmask8)0xff, z0); +} + +XALIGNED(16) static const word32 L_mldsa_encode_eta_2_x2_avx512_two[] + WC_X64I_UNUSED = { + 0x00000002, 0x00000002, 0x00000002, 0x00000002, + 0x00000002, 0x00000002, 0x00000002, 0x00000002, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_eta_2_x2_avx512_vs_3[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000003, 0x00000006, 0x00000009, + 0x00000004, 0x00000007, 0x0000000a, 0x0000000d, +}; + +XALIGNED(16) static const byte L_mldsa_encode_eta_2_x2_avx512_shuff_3_even[] + WC_X64I_UNUSED = { + 0x00, 0xff, 0x04, 0x05, 0x08, 0xff, 0x0c, 0x0d, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0xff, 0x04, 0x05, 0x08, 0xff, 0x0c, 0x0d, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_eta_2_x2_avx512_shuff_3_odd[] + WC_X64I_UNUSED = { + 0x02, 0xff, 0xff, 0x07, 0x0a, 0x0b, 0xff, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x02, 0xff, 0xff, 0x07, 0x0a, 0x0b, 0xff, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_eta_2_x2_avx512_shuff_6_even[] + WC_X64I_UNUSED = { + 0x00, 0x04, 0x05, 0x08, 0x0c, 0x0d, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x04, + 0x05, 0x08, 0x0c, 0x0d, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_eta_2_x2_avx512_shuff_6_odd[] + WC_X64I_UNUSED = { + 0x02, 0x03, 0x07, 0x0a, 0x0b, 0x0f, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x03, + 0x07, 0x0a, 0x0b, 0x0f, 0xff, 0xff, 0xff, 0xff, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void wc_mldsa_encode_eta_2_x2_avx512(const sword32* sA, + const sword32* sB, byte* pA, byte* pB) +{ + word64 rdi, rsi, rdx, rcx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11; + + rdi = (word64)(size_t)sA; + rsi = (word64)(size_t)sB; + rdx = (word64)(size_t)pA; + rcx = (word64)(size_t)pB; + + z6 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_x2_avx512_two, 0))); + z7 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_x2_avx512_vs_3, 0))); + z8 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_x2_avx512_shuff_3_even, 0))); + z9 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_x2_avx512_shuff_3_odd, 0))); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_x2_avx512_shuff_6_even, 0))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_eta_2_x2_avx512_shuff_6_odd, 0))); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z0 = _mm512_sub_epi32(z6, z0); + z1 = _mm512_sub_epi32(z6, z1); + z2 = _mm512_sub_epi32(z6, z2); + z3 = _mm512_sub_epi32(z6, z3); + z0 = _mm512_sllv_epi32(z0, z7); + z1 = _mm512_sllv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z7); + z3 = _mm512_sllv_epi32(z3, z7); + z0 = _mm512_packus_epi32(z0, z2); + z1 = _mm512_packus_epi32(z1, z3); + z0 = _mm512_permutex_epi64(z0, 0xd8); + z1 = _mm512_permutex_epi64(z1, 0xd8); + z4 = _mm512_shuffle_epi8(z0, z9); + z5 = _mm512_shuffle_epi8(z1, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z1 = _mm512_shuffle_epi8(z1, z8); + z0 = _mm512_or_si512(z0, z4); + z1 = _mm512_or_si512(z1, z5); + z1 = _mm512_permutex_epi64(z1, 0xb1); + z0 = _mm512_or_si512(z0, z1); + z4 = _mm512_shuffle_epi8(z0, z11); + z0 = _mm512_shuffle_epi8(z0, z10); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z4)); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z0 = _mm512_sub_epi32(z6, z0); + z1 = _mm512_sub_epi32(z6, z1); + z2 = _mm512_sub_epi32(z6, z2); + z3 = _mm512_sub_epi32(z6, z3); + z0 = _mm512_sllv_epi32(z0, z7); + z1 = _mm512_sllv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z7); + z3 = _mm512_sllv_epi32(z3, z7); + z0 = _mm512_packus_epi32(z0, z2); + z1 = _mm512_packus_epi32(z1, z3); + z0 = _mm512_permutex_epi64(z0, 0xd8); + z1 = _mm512_permutex_epi64(z1, 0xd8); + z4 = _mm512_shuffle_epi8(z0, z9); + z5 = _mm512_shuffle_epi8(z1, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z1 = _mm512_shuffle_epi8(z1, z8); + z0 = _mm512_or_si512(z0, z4); + z1 = _mm512_or_si512(z1, z5); + z1 = _mm512_permutex_epi64(z1, 0xb1); + z0 = _mm512_or_si512(z0, z1); + z4 = _mm512_shuffle_epi8(z0, z11); + z0 = _mm512_shuffle_epi8(z0, z10); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z4)); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z0 = _mm512_sub_epi32(z6, z0); + z1 = _mm512_sub_epi32(z6, z1); + z2 = _mm512_sub_epi32(z6, z2); + z3 = _mm512_sub_epi32(z6, z3); + z0 = _mm512_sllv_epi32(z0, z7); + z1 = _mm512_sllv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z7); + z3 = _mm512_sllv_epi32(z3, z7); + z0 = _mm512_packus_epi32(z0, z2); + z1 = _mm512_packus_epi32(z1, z3); + z0 = _mm512_permutex_epi64(z0, 0xd8); + z1 = _mm512_permutex_epi64(z1, 0xd8); + z4 = _mm512_shuffle_epi8(z0, z9); + z5 = _mm512_shuffle_epi8(z1, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z1 = _mm512_shuffle_epi8(z1, z8); + z0 = _mm512_or_si512(z0, z4); + z1 = _mm512_or_si512(z1, z5); + z1 = _mm512_permutex_epi64(z1, 0xb1); + z0 = _mm512_or_si512(z0, z1); + z4 = _mm512_shuffle_epi8(z0, z11); + z0 = _mm512_shuffle_epi8(z0, z10); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z4)); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z0 = _mm512_sub_epi32(z6, z0); + z1 = _mm512_sub_epi32(z6, z1); + z2 = _mm512_sub_epi32(z6, z2); + z3 = _mm512_sub_epi32(z6, z3); + z0 = _mm512_sllv_epi32(z0, z7); + z1 = _mm512_sllv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z7); + z3 = _mm512_sllv_epi32(z3, z7); + z0 = _mm512_packus_epi32(z0, z2); + z1 = _mm512_packus_epi32(z1, z3); + z0 = _mm512_permutex_epi64(z0, 0xd8); + z1 = _mm512_permutex_epi64(z1, 0xd8); + z4 = _mm512_shuffle_epi8(z0, z9); + z5 = _mm512_shuffle_epi8(z1, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z1 = _mm512_shuffle_epi8(z1, z8); + z0 = _mm512_or_si512(z0, z4); + z1 = _mm512_or_si512(z1, z5); + z1 = _mm512_permutex_epi64(z1, 0xb1); + z0 = _mm512_or_si512(z0, z1); + z4 = _mm512_shuffle_epi8(z0, z11); + z0 = _mm512_shuffle_epi8(z0, z10); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z4)); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 512))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 512)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 544))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 544)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 576))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 576)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 608))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 608)), 1); + z0 = _mm512_sub_epi32(z6, z0); + z1 = _mm512_sub_epi32(z6, z1); + z2 = _mm512_sub_epi32(z6, z2); + z3 = _mm512_sub_epi32(z6, z3); + z0 = _mm512_sllv_epi32(z0, z7); + z1 = _mm512_sllv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z7); + z3 = _mm512_sllv_epi32(z3, z7); + z0 = _mm512_packus_epi32(z0, z2); + z1 = _mm512_packus_epi32(z1, z3); + z0 = _mm512_permutex_epi64(z0, 0xd8); + z1 = _mm512_permutex_epi64(z1, 0xd8); + z4 = _mm512_shuffle_epi8(z0, z9); + z5 = _mm512_shuffle_epi8(z1, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z1 = _mm512_shuffle_epi8(z1, z8); + z0 = _mm512_or_si512(z0, z4); + z1 = _mm512_or_si512(z1, z5); + z1 = _mm512_permutex_epi64(z1, 0xb1); + z0 = _mm512_or_si512(z0, z1); + z4 = _mm512_shuffle_epi8(z0, z11); + z0 = _mm512_shuffle_epi8(z0, z10); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z4)); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 640))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 640)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 672))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 672)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 704))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 704)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 736))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 736)), 1); + z0 = _mm512_sub_epi32(z6, z0); + z1 = _mm512_sub_epi32(z6, z1); + z2 = _mm512_sub_epi32(z6, z2); + z3 = _mm512_sub_epi32(z6, z3); + z0 = _mm512_sllv_epi32(z0, z7); + z1 = _mm512_sllv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z7); + z3 = _mm512_sllv_epi32(z3, z7); + z0 = _mm512_packus_epi32(z0, z2); + z1 = _mm512_packus_epi32(z1, z3); + z0 = _mm512_permutex_epi64(z0, 0xd8); + z1 = _mm512_permutex_epi64(z1, 0xd8); + z4 = _mm512_shuffle_epi8(z0, z9); + z5 = _mm512_shuffle_epi8(z1, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z1 = _mm512_shuffle_epi8(z1, z8); + z0 = _mm512_or_si512(z0, z4); + z1 = _mm512_or_si512(z1, z5); + z1 = _mm512_permutex_epi64(z1, 0xb1); + z0 = _mm512_or_si512(z0, z1); + z4 = _mm512_shuffle_epi8(z0, z11); + z0 = _mm512_shuffle_epi8(z0, z10); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z4)); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 768))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 768)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 800))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 800)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 832))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 832)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 864))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 864)), 1); + z0 = _mm512_sub_epi32(z6, z0); + z1 = _mm512_sub_epi32(z6, z1); + z2 = _mm512_sub_epi32(z6, z2); + z3 = _mm512_sub_epi32(z6, z3); + z0 = _mm512_sllv_epi32(z0, z7); + z1 = _mm512_sllv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z7); + z3 = _mm512_sllv_epi32(z3, z7); + z0 = _mm512_packus_epi32(z0, z2); + z1 = _mm512_packus_epi32(z1, z3); + z0 = _mm512_permutex_epi64(z0, 0xd8); + z1 = _mm512_permutex_epi64(z1, 0xd8); + z4 = _mm512_shuffle_epi8(z0, z9); + z5 = _mm512_shuffle_epi8(z1, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z1 = _mm512_shuffle_epi8(z1, z8); + z0 = _mm512_or_si512(z0, z4); + z1 = _mm512_or_si512(z1, z5); + z1 = _mm512_permutex_epi64(z1, 0xb1); + z0 = _mm512_or_si512(z0, z1); + z4 = _mm512_shuffle_epi8(z0, z11); + z0 = _mm512_shuffle_epi8(z0, z10); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z4)); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 896))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 896)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 928))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 928)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 960))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 960)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 992))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 992)), 1); + z0 = _mm512_sub_epi32(z6, z0); + z1 = _mm512_sub_epi32(z6, z1); + z2 = _mm512_sub_epi32(z6, z2); + z3 = _mm512_sub_epi32(z6, z3); + z0 = _mm512_sllv_epi32(z0, z7); + z1 = _mm512_sllv_epi32(z1, z7); + z2 = _mm512_sllv_epi32(z2, z7); + z3 = _mm512_sllv_epi32(z3, z7); + z0 = _mm512_packus_epi32(z0, z2); + z1 = _mm512_packus_epi32(z1, z3); + z0 = _mm512_permutex_epi64(z0, 0xd8); + z1 = _mm512_permutex_epi64(z1, 0xd8); + z4 = _mm512_shuffle_epi8(z0, z9); + z5 = _mm512_shuffle_epi8(z1, z9); + z0 = _mm512_shuffle_epi8(z0, z8); + z1 = _mm512_shuffle_epi8(z1, z8); + z0 = _mm512_or_si512(z0, z4); + z1 = _mm512_or_si512(z1, z5); + z1 = _mm512_permutex_epi64(z1, 0xb1); + z0 = _mm512_or_si512(z0, z1); + z4 = _mm512_shuffle_epi8(z0, z11); + z0 = _mm512_shuffle_epi8(z0, z10); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_shuffle_i64x2(z0, z0, 0xf5); + z0 = _mm512_or_si512(z0, z4); + z4 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z0, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z0)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z0), 2); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z4)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z4), 2); + rdx = (word64)(rdx + 0xc); + rcx = (word64)(rcx + 0xc); +} + +XALIGNED(16) static const word32 L_mldsa_encode_eta_4_avx512_four[] + WC_X64I_UNUSED = { + 0x00000004, 0x00000004, 0x00000004, 0x00000004, + 0x00000004, 0x00000004, 0x00000004, 0x00000004, + 0x00000004, 0x00000004, 0x00000004, 0x00000004, + 0x00000004, 0x00000004, 0x00000004, 0x00000004, +}; + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void wc_mldsa_encode_eta_4_avx512(const sword32* s, byte* p) +{ + word64 rdi, rsi; + __m512i z0, z1, z2; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)p; + + z2 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_encode_eta_4_avx512_four, + 0)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 0), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 8), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 16), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 24), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 32), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 40), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 48), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 56), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 64), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 72), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 80), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 88), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 96), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 104), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 112), (__mmask8)0xff, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z0 = _mm512_sub_epi32(z2, z0); + z1 = _mm512_srli_epi64(z0, 28); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_cvtepi64_storeu_epi8(WC_PW(rsi, 120), (__mmask8)0xff, z0); +} + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_x2_avx512_d_max_half_m1[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_x2_avx512_d_max_half[] + WC_X64I_UNUSED = { + 0x00001000, 0x00001000, 0x00001000, 0x00001000, + 0x00001000, 0x00001000, 0x00001000, 0x00001000, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_x2_avx512_vs_13[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000d, 0x00000002, 0x0000000f, + 0x00000004, 0x00000011, 0x00000006, 0x00000013, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_x2_avx512_shuff_13_even[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0xff, 0x08, 0x09, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, + 0x02, 0x08, 0x09, 0x0a, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_x2_avx512_shuff_13_odd[] + WC_X64I_UNUSED = { + 0xff, 0x05, 0x06, 0x07, 0x0d, 0x0e, 0x0f, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x06, 0x07, 0xff, 0x0e, 0x0f, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_x2_avx512_vs_10[] + WC_X64I_UNUSED = { + 0x00000000, 0x0000000a, 0x00000004, 0x0000000e, + 0x00000000, 0x0000000a, 0x00000004, 0x0000000e, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_x2_avx512_shuff_10_even[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x08, 0x09, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x08, + 0x09, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_x2_avx512_shuff_10_odd[] + WC_X64I_UNUSED = { + 0xff, 0x05, 0x06, 0x0d, 0x0e, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x05, 0x06, + 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void wc_mldsa_encode_t0_t1_x2_avx512(const sword32* tA, + const sword32* tB, byte* t0A, byte* t0B, byte* t1A, byte* t1B) +{ + word64 rdi, rsi, rdx, rcx, r8, r9; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13; + + rdi = (word64)(size_t)tA; + rsi = (word64)(size_t)tB; + rdx = (word64)(size_t)t0A; + rcx = (word64)(size_t)t0B; + r8 = (word64)(size_t)t1A; + r9 = (word64)(size_t)t1B; + + z6 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_x2_avx512_d_max_half_m1, 0))); + z7 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_x2_avx512_d_max_half, 0))); + z8 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_x2_avx512_vs_13, 0))); + z9 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_x2_avx512_shuff_13_even, 0))); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_x2_avx512_shuff_13_odd, 0))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_x2_avx512_vs_10, 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_x2_avx512_shuff_10_even, 0))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR( + L_mldsa_encode_t0_t1_x2_avx512_shuff_10_odd, 0))); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 512))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 512)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 544))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 544)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 576))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 576)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 608))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 608)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 640))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 640)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 672))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 672)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 704))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 704)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 736))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 736)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 768))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 768)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 800))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 800)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 832))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 832)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 864))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 864)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 896))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 896)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 928))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 928)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 960))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 960)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 992))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 992)), 1); + z4 = _mm512_add_epi32(z0, z6); + z5 = _mm512_add_epi32(z1, z6); + z4 = _mm512_srli_epi32(z4, 13); + z5 = _mm512_srli_epi32(z5, 13); + z2 = _mm512_slli_epi32(z4, 13); + z3 = _mm512_slli_epi32(z5, 13); + z2 = _mm512_sub_epi32(z0, z2); + z3 = _mm512_sub_epi32(z1, z3); + z2 = _mm512_sub_epi32(z7, z2); + z3 = _mm512_sub_epi32(z7, z3); + z2 = _mm512_sllv_epi32(z2, z8); + z3 = _mm512_sllv_epi32(z3, z8); + z0 = _mm512_shuffle_epi8(z2, z10); + z1 = _mm512_shuffle_epi8(z3, z10); + z2 = _mm512_shuffle_epi8(z2, z9); + z3 = _mm512_shuffle_epi8(z3, z9); + z2 = _mm512_or_si512(z2, z0); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_shuffle_i64x2(z2, z2, 0xf5); + z2 = _mm512_or_si512(z2, z0); + z1 = _mm512_shuffle_i64x2(z3, z3, 0xf5); + z3 = _mm512_or_si512(z3, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z2, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z2)); + _mm_storeu_si128((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z0)); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z3, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdx, 0), _mm512_castsi512_si128(z3)); + WC_S32(rdx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z3), 2); + WC_S8(rdx, 12) = (word8)_mm_extract_epi8(_mm512_castsi512_si128(z3), 12); + _mm_storel_epi64((__m128i*)WC_PW(rcx, 0), _mm512_castsi512_si128(z1)); + WC_S32(rcx, 8) = (word32)_mm_extract_epi32(_mm512_castsi512_si128(z1), 2); + WC_S8(rcx, 12) = (word8)_mm_extract_epi8(_mm512_castsi512_si128(z1), 12); + rdx = (word64)(rdx + 0xd); + rcx = (word64)(rcx + 0xd); + z4 = _mm512_sllv_epi32(z4, z11); + z5 = _mm512_sllv_epi32(z5, z11); + z0 = _mm512_shuffle_epi8(z4, z13); + z1 = _mm512_shuffle_epi8(z5, z13); + z4 = _mm512_shuffle_epi8(z4, z12); + z5 = _mm512_shuffle_epi8(z5, z12); + z4 = _mm512_or_si512(z4, z0); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_shuffle_i64x2(z4, z4, 0xf5); + z4 = _mm512_or_si512(z4, z0); + z1 = _mm512_shuffle_i64x2(z5, z5, 0xf5); + z5 = _mm512_or_si512(z5, z1); + z0 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z4, 2)); + _mm_storeu_si128((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z4)); + _mm_storeu_si128((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z0)); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); + z1 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z5, 2)); + _mm_storel_epi64((__m128i*)WC_PW(r8, 0), _mm512_castsi512_si128(z5)); + WC_S16(r8, 8) = (word16)_mm_extract_epi16(_mm512_castsi512_si128(z5), 4); + _mm_storel_epi64((__m128i*)WC_PW(r9, 0), _mm512_castsi512_si128(z1)); + WC_S16(r9, 8) = (word16)_mm_extract_epi16(_mm512_castsi512_si128(z1), 4); + r8 = (word64)(r8 + 0xa); + r9 = (word64)(r9 + 0xa); +} + +XALIGNED(16) static const word32 L_mldsa_avx512_zetas_1p[] WC_X64I_UNUSED = { + 0x000064f7, 0x000064f7, 0x000064f7, 0x000064f7, + 0x000064f7, 0x000064f7, 0x000064f7, 0x000064f7, + 0x000064f7, 0x000064f7, 0x000064f7, 0x000064f7, + 0x000064f7, 0x000064f7, 0x000064f7, 0x000064f7, + 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, + 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, + 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, + 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, 0x6d1f44f7, + 0xffd83102, 0xffd83102, 0xffd83102, 0xffd83102, + 0xffd83102, 0xffd83102, 0xffd83102, 0xffd83102, + 0xffd83102, 0xffd83102, 0xffd83102, 0xffd83102, + 0xffd83102, 0xffd83102, 0xffd83102, 0xffd83102, + 0x8cf87102, 0x8cf87102, 0x8cf87102, 0x8cf87102, + 0x8cf87102, 0x8cf87102, 0x8cf87102, 0x8cf87102, + 0x8cf87102, 0x8cf87102, 0x8cf87102, 0x8cf87102, + 0x8cf87102, 0x8cf87102, 0x8cf87102, 0x8cf87102, + 0xfff81503, 0xfff81503, 0xfff81503, 0xfff81503, + 0xfff81503, 0xfff81503, 0xfff81503, 0xfff81503, + 0xfff81503, 0xfff81503, 0xfff81503, 0xfff81503, + 0xfff81503, 0xfff81503, 0xfff81503, 0xfff81503, + 0x8d187503, 0x8d187503, 0x8d187503, 0x8d187503, + 0x8d187503, 0x8d187503, 0x8d187503, 0x8d187503, + 0x8d187503, 0x8d187503, 0x8d187503, 0x8d187503, + 0x8d187503, 0x8d187503, 0x8d187503, 0x8d187503, + 0x00039e44, 0x00039e44, 0x00039e44, 0x00039e44, + 0x00039e44, 0x00039e44, 0x00039e44, 0x00039e44, + 0x00039e44, 0x00039e44, 0x00039e44, 0x00039e44, + 0x00039e44, 0x00039e44, 0x00039e44, 0x00039e44, + 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, + 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, + 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, + 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, 0x61cc1e44, + 0xfff42118, 0xfff42118, 0xfff42118, 0xfff42118, + 0xfff42118, 0xfff42118, 0xfff42118, 0xfff42118, + 0xfff42118, 0xfff42118, 0xfff42118, 0xfff42118, + 0xfff42118, 0xfff42118, 0xfff42118, 0xfff42118, + 0x58172118, 0x58172118, 0x58172118, 0x58172118, + 0x58172118, 0x58172118, 0x58172118, 0x58172118, + 0x58172118, 0x58172118, 0x58172118, 0x58172118, + 0x58172118, 0x58172118, 0x58172118, 0x58172118, + 0xfff2a128, 0xfff2a128, 0xfff2a128, 0xfff2a128, + 0xfff2a128, 0xfff2a128, 0xfff2a128, 0xfff2a128, + 0xfff2a128, 0xfff2a128, 0xfff2a128, 0xfff2a128, + 0xfff2a128, 0xfff2a128, 0xfff2a128, 0xfff2a128, + 0x6017a128, 0x6017a128, 0x6017a128, 0x6017a128, + 0x6017a128, 0x6017a128, 0x6017a128, 0x6017a128, + 0x6017a128, 0x6017a128, 0x6017a128, 0x6017a128, + 0x6017a128, 0x6017a128, 0x6017a128, 0x6017a128, + 0x00071e24, 0x00071e24, 0x00071e24, 0x00071e24, + 0x00071e24, 0x00071e24, 0x00071e24, 0x00071e24, + 0x00071e24, 0x00071e24, 0x00071e24, 0x00071e24, + 0x00071e24, 0x00071e24, 0x00071e24, 0x00071e24, + 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, + 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, + 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, + 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, 0x61cb9e24, + 0x001bde2b, 0x001bde2b, 0x001bde2b, 0x001bde2b, + 0x001bde2b, 0x001bde2b, 0x001bde2b, 0x001bde2b, + 0x001bde2b, 0x001bde2b, 0x001bde2b, 0x001bde2b, + 0x001bde2b, 0x001bde2b, 0x001bde2b, 0x001bde2b, + 0x12613e2b, 0x12613e2b, 0x12613e2b, 0x12613e2b, + 0x12613e2b, 0x12613e2b, 0x12613e2b, 0x12613e2b, + 0x12613e2b, 0x12613e2b, 0x12613e2b, 0x12613e2b, + 0x12613e2b, 0x12613e2b, 0x12613e2b, 0x12613e2b, + 0x0023e92b, 0x0023e92b, 0x0023e92b, 0x0023e92b, + 0x0023e92b, 0x0023e92b, 0x0023e92b, 0x0023e92b, + 0x0023e92b, 0x0023e92b, 0x0023e92b, 0x0023e92b, + 0x0023e92b, 0x0023e92b, 0x0023e92b, 0x0023e92b, + 0x93c9492b, 0x93c9492b, 0x93c9492b, 0x93c9492b, + 0x93c9492b, 0x93c9492b, 0x93c9492b, 0x93c9492b, + 0x93c9492b, 0x93c9492b, 0x93c9492b, 0x93c9492b, + 0x93c9492b, 0x93c9492b, 0x93c9492b, 0x93c9492b, + 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, + 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, + 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, + 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, 0xfffa84ad, + 0xae1024ad, 0xae1024ad, 0xae1024ad, 0xae1024ad, + 0xae1024ad, 0xae1024ad, 0xae1024ad, 0xae1024ad, + 0xae1024ad, 0xae1024ad, 0xae1024ad, 0xae1024ad, + 0xae1024ad, 0xae1024ad, 0xae1024ad, 0xae1024ad, + 0xffe0147f, 0xffe0147f, 0xffe0147f, 0xffe0147f, + 0xffe0147f, 0xffe0147f, 0xffe0147f, 0xffe0147f, + 0xffe0147f, 0xffe0147f, 0xffe0147f, 0xffe0147f, + 0xffe0147f, 0xffe0147f, 0xffe0147f, 0xffe0147f, + 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, + 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, + 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, + 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, 0xbeeff47f, + 0x002f9a75, 0x002f9a75, 0x002f9a75, 0x002f9a75, + 0x002f9a75, 0x002f9a75, 0x002f9a75, 0x002f9a75, + 0x002f9a75, 0x002f9a75, 0x002f9a75, 0x002f9a75, + 0x002f9a75, 0x002f9a75, 0x002f9a75, 0x002f9a75, + 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, + 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, + 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, + 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, 0x8cfe3a75, + 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, + 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, + 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, + 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, 0xffd3fb09, + 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, + 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, + 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, + 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, 0x1eb51b09, + 0x002f7a49, 0x002f7a49, 0x002f7a49, 0x002f7a49, + 0x002f7a49, 0x002f7a49, 0x002f7a49, 0x002f7a49, + 0x002f7a49, 0x002f7a49, 0x002f7a49, 0x002f7a49, + 0x002f7a49, 0x002f7a49, 0x002f7a49, 0x002f7a49, + 0xeef89a49, 0xeef89a49, 0xeef89a49, 0xeef89a49, + 0xeef89a49, 0xeef89a49, 0xeef89a49, 0xeef89a49, + 0xeef89a49, 0xeef89a49, 0xeef89a49, 0xeef89a49, + 0xeef89a49, 0xeef89a49, 0xeef89a49, 0xeef89a49, + 0x0028e527, 0x0028e527, 0x0028e527, 0x0028e527, + 0x0028e527, 0x0028e527, 0x0028e527, 0x0028e527, + 0x0028e527, 0x0028e527, 0x0028e527, 0x0028e527, + 0x0028e527, 0x0028e527, 0x0028e527, 0x0028e527, + 0x254dc527, 0x254dc527, 0x254dc527, 0x254dc527, + 0x254dc527, 0x254dc527, 0x254dc527, 0x254dc527, + 0x254dc527, 0x254dc527, 0x254dc527, 0x254dc527, + 0x254dc527, 0x254dc527, 0x254dc527, 0x254dc527, + 0x00299658, 0x00299658, 0x00299658, 0x00299658, + 0x00299658, 0x00299658, 0x00299658, 0x00299658, + 0x000fa070, 0x000fa070, 0x000fa070, 0x000fa070, + 0x000fa070, 0x000fa070, 0x000fa070, 0x000fa070, + 0x66f49658, 0x66f49658, 0x66f49658, 0x66f49658, + 0x66f49658, 0x66f49658, 0x66f49658, 0x66f49658, + 0x7c1da070, 0x7c1da070, 0x7c1da070, 0x7c1da070, + 0x7c1da070, 0x7c1da070, 0x7c1da070, 0x7c1da070, + 0xffef85a4, 0xffef85a4, 0xffef85a4, 0xffef85a4, + 0xffef85a4, 0xffef85a4, 0xffef85a4, 0xffef85a4, + 0x0036b788, 0x0036b788, 0x0036b788, 0x0036b788, + 0x0036b788, 0x0036b788, 0x0036b788, 0x0036b788, + 0xaea405a4, 0xaea405a4, 0xaea405a4, 0xaea405a4, + 0xaea405a4, 0xaea405a4, 0xaea405a4, 0xaea405a4, + 0x3327b788, 0x3327b788, 0x3327b788, 0x3327b788, + 0x3327b788, 0x3327b788, 0x3327b788, 0x3327b788, + 0xfff79d90, 0xfff79d90, 0xfff79d90, 0xfff79d90, + 0xfff79d90, 0xfff79d90, 0xfff79d90, 0xfff79d90, + 0xffeeeaa0, 0xffeeeaa0, 0xffeeeaa0, 0xffeeeaa0, + 0xffeeeaa0, 0xffeeeaa0, 0xffeeeaa0, 0xffeeeaa0, + 0x6ba99d90, 0x6ba99d90, 0x6ba99d90, 0x6ba99d90, + 0x6ba99d90, 0x6ba99d90, 0x6ba99d90, 0x6ba99d90, + 0x0d42eaa0, 0x0d42eaa0, 0x0d42eaa0, 0x0d42eaa0, + 0x0d42eaa0, 0x0d42eaa0, 0x0d42eaa0, 0x0d42eaa0, + 0x0027f968, 0x0027f968, 0x0027f968, 0x0027f968, + 0x0027f968, 0x0027f968, 0x0027f968, 0x0027f968, + 0xffdfd37b, 0xffdfd37b, 0xffdfd37b, 0xffdfd37b, + 0xffdfd37b, 0xffdfd37b, 0xffdfd37b, 0xffdfd37b, + 0xeb54f968, 0xeb54f968, 0xeb54f968, 0xeb54f968, + 0xeb54f968, 0xeb54f968, 0xeb54f968, 0xeb54f968, + 0x28cf337b, 0x28cf337b, 0x28cf337b, 0x28cf337b, + 0x28cf337b, 0x28cf337b, 0x28cf337b, 0x28cf337b, + 0xffdfadd6, 0xffdfadd6, 0xffdfadd6, 0xffdfadd6, + 0xffdfadd6, 0xffdfadd6, 0xffdfadd6, 0xffdfadd6, + 0xffc51ae7, 0xffc51ae7, 0xffc51ae7, 0xffc51ae7, + 0xffc51ae7, 0xffc51ae7, 0xffc51ae7, 0xffc51ae7, + 0x629a6dd6, 0x629a6dd6, 0x629a6dd6, 0x629a6dd6, + 0x629a6dd6, 0x629a6dd6, 0x629a6dd6, 0x629a6dd6, + 0xcba1fae7, 0xcba1fae7, 0xcba1fae7, 0xcba1fae7, + 0xcba1fae7, 0xcba1fae7, 0xcba1fae7, 0xcba1fae7, + 0xffeaa4f7, 0xffeaa4f7, 0xffeaa4f7, 0xffeaa4f7, + 0xffeaa4f7, 0xffeaa4f7, 0xffeaa4f7, 0xffeaa4f7, + 0xffcdfc98, 0xffcdfc98, 0xffcdfc98, 0xffcdfc98, + 0xffcdfc98, 0xffcdfc98, 0xffcdfc98, 0xffcdfc98, + 0xb50984f7, 0xb50984f7, 0xb50984f7, 0xb50984f7, + 0xb50984f7, 0xb50984f7, 0xb50984f7, 0xb50984f7, + 0xd360fc98, 0xd360fc98, 0xd360fc98, 0xd360fc98, + 0xd360fc98, 0xd360fc98, 0xd360fc98, 0xd360fc98, + 0x001ad035, 0x001ad035, 0x001ad035, 0x001ad035, + 0x001ad035, 0x001ad035, 0x001ad035, 0x001ad035, + 0xffffb422, 0xffffb422, 0xffffb422, 0xffffb422, + 0xffffb422, 0xffffb422, 0xffffb422, 0xffffb422, + 0x13a17035, 0x13a17035, 0x13a17035, 0x13a17035, + 0x13a17035, 0x13a17035, 0x13a17035, 0x13a17035, + 0x6d83f422, 0x6d83f422, 0x6d83f422, 0x6d83f422, + 0x6d83f422, 0x6d83f422, 0x6d83f422, 0x6d83f422, + 0x003d3201, 0x003d3201, 0x003d3201, 0x003d3201, + 0x003d3201, 0x003d3201, 0x003d3201, 0x003d3201, + 0x000445c5, 0x000445c5, 0x000445c5, 0x000445c5, + 0x000445c5, 0x000445c5, 0x000445c5, 0x000445c5, + 0xa9fd5201, 0xa9fd5201, 0xa9fd5201, 0xa9fd5201, + 0xa9fd5201, 0xa9fd5201, 0xa9fd5201, 0xa9fd5201, + 0xba3ce5c5, 0xba3ce5c5, 0xba3ce5c5, 0xba3ce5c5, + 0xba3ce5c5, 0xba3ce5c5, 0xba3ce5c5, 0xba3ce5c5, + 0x00294a67, 0x00294a67, 0x00294a67, 0x00294a67, + 0x00017620, 0x00017620, 0x00017620, 0x00017620, + 0x002ef4cd, 0x002ef4cd, 0x002ef4cd, 0x002ef4cd, + 0x0035dec5, 0x0035dec5, 0x0035dec5, 0x0035dec5, + 0x91f62a67, 0x91f62a67, 0x91f62a67, 0x91f62a67, + 0x9ec57620, 0x9ec57620, 0x9ec57620, 0x9ec57620, + 0xac4894cd, 0xac4894cd, 0xac4894cd, 0xac4894cd, + 0x6d8e7ec5, 0x6d8e7ec5, 0x6d8e7ec5, 0x6d8e7ec5, + 0xffe6a503, 0xffe6a503, 0xffe6a503, 0xffe6a503, + 0xffc9302c, 0xffc9302c, 0xffc9302c, 0xffc9302c, + 0xffd947d4, 0xffd947d4, 0xffd947d4, 0xffd947d4, + 0x003bbeaf, 0x003bbeaf, 0x003bbeaf, 0x003bbeaf, + 0x5f070503, 0x5f070503, 0x5f070503, 0x5f070503, + 0xbfceb02c, 0xbfceb02c, 0xbfceb02c, 0xbfceb02c, + 0x8ed3c7d4, 0x8ed3c7d4, 0x8ed3c7d4, 0x8ed3c7d4, + 0xdc919eaf, 0xdc919eaf, 0xdc919eaf, 0xdc919eaf, + 0xffc51585, 0xffc51585, 0xffc51585, 0xffc51585, + 0xffd18e7c, 0xffd18e7c, 0xffd18e7c, 0xffd18e7c, + 0x00368a96, 0x00368a96, 0x00368a96, 0x00368a96, + 0xffd43e41, 0xffd43e41, 0xffd43e41, 0xffd43e41, + 0xf3f5b585, 0xf3f5b585, 0xf3f5b585, 0xf3f5b585, + 0xe3a10e7c, 0xe3a10e7c, 0xe3a10e7c, 0xe3a10e7c, + 0xde894a96, 0xde894a96, 0xde894a96, 0xde894a96, + 0x6b1c5e41, 0x6b1c5e41, 0x6b1c5e41, 0x6b1c5e41, + 0x00360400, 0x00360400, 0x00360400, 0x00360400, + 0xfffb6a4d, 0xfffb6a4d, 0xfffb6a4d, 0xfffb6a4d, + 0x0023d69c, 0x0023d69c, 0x0023d69c, 0x0023d69c, + 0xfff7c55d, 0xfff7c55d, 0xfff7c55d, 0xfff7c55d, + 0xc0b60400, 0xc0b60400, 0xc0b60400, 0xc0b60400, + 0x7ac50a4d, 0x7ac50a4d, 0x7ac50a4d, 0x7ac50a4d, + 0x9cf7569c, 0x9cf7569c, 0x9cf7569c, 0x9cf7569c, + 0xbe23655d, 0xbe23655d, 0xbe23655d, 0xbe23655d, + 0xffe6123d, 0xffe6123d, 0xffe6123d, 0xffe6123d, + 0xffe6ead6, 0xffe6ead6, 0xffe6ead6, 0xffe6ead6, + 0x00357e1e, 0x00357e1e, 0x00357e1e, 0x00357e1e, + 0xffc5af59, 0xffc5af59, 0xffc5af59, 0xffc5af59, + 0x97adb23d, 0x97adb23d, 0x97adb23d, 0x97adb23d, + 0xca41aad6, 0xca41aad6, 0xca41aad6, 0xca41aad6, + 0x18f93e1e, 0x18f93e1e, 0x18f93e1e, 0x18f93e1e, + 0x6d30cf59, 0x6d30cf59, 0x6d30cf59, 0x6d30cf59, + 0x0035843f, 0x0035843f, 0x0035843f, 0x0035843f, + 0xffdf5617, 0xffdf5617, 0xffdf5617, 0xffdf5617, + 0xffe7945c, 0xffe7945c, 0xffe7945c, 0xffe7945c, + 0x0038738c, 0x0038738c, 0x0038738c, 0x0038738c, + 0x8d3d643f, 0x8d3d643f, 0x8d3d643f, 0x8d3d643f, + 0x3b223617, 0x3b223617, 0x3b223617, 0x3b223617, + 0x3473145c, 0x3473145c, 0x3473145c, 0x3473145c, + 0x78a9f38c, 0x78a9f38c, 0x78a9f38c, 0x78a9f38c, + 0x000c63a8, 0x000c63a8, 0x000c63a8, 0x000c63a8, + 0x00081b9a, 0x00081b9a, 0x00081b9a, 0x00081b9a, + 0x000e8f76, 0x000e8f76, 0x000e8f76, 0x000e8f76, + 0x003b3853, 0x003b3853, 0x003b3853, 0x003b3853, + 0x588163a8, 0x588163a8, 0x588163a8, 0x588163a8, + 0x9e7b5b9a, 0x9e7b5b9a, 0x9e7b5b9a, 0x9e7b5b9a, + 0xeefd4f76, 0xeefd4f76, 0xeefd4f76, 0xeefd4f76, + 0x89c59853, 0x89c59853, 0x89c59853, 0x89c59853, + 0x003b8534, 0x003b8534, 0x003b8534, 0x003b8534, + 0xffd8fc30, 0xffd8fc30, 0xffd8fc30, 0xffd8fc30, + 0x001f9d54, 0x001f9d54, 0x001f9d54, 0x001f9d54, + 0xffd54f2d, 0xffd54f2d, 0xffd54f2d, 0xffd54f2d, + 0xa6e20534, 0xa6e20534, 0xa6e20534, 0xa6e20534, + 0xc75efc30, 0xc75efc30, 0xc75efc30, 0xc75efc30, + 0x99ca1d54, 0x99ca1d54, 0x99ca1d54, 0x99ca1d54, + 0xc73aef2d, 0xc73aef2d, 0xc73aef2d, 0xc73aef2d, + 0xffc406e5, 0xffc406e5, 0xffe8ac81, 0xffe8ac81, + 0xffc7e1cf, 0xffc7e1cf, 0xffd19819, 0xffd19819, + 0xffe9d65d, 0xffe9d65d, 0x003509ee, 0x003509ee, + 0x002135c7, 0x002135c7, 0xffe7cfbb, 0xffe7cfbb, + 0xa220a6e5, 0xa220a6e5, 0xd8f8cc81, 0xd8f8cc81, + 0x5081c1cf, 0x5081c1cf, 0x8a54b819, 0x8a54b819, + 0x8035765d, 0x8035765d, 0x6272c9ee, 0x6272c9ee, + 0x5f5a15c7, 0x5f5a15c7, 0x085f2fbb, 0x085f2fbb, + 0xffeccf75, 0xffeccf75, 0x001d9772, 0x001d9772, + 0xffc1b072, 0xffc1b072, 0xfff0bcf6, 0xfff0bcf6, + 0xffcf5280, 0xffcf5280, 0xffcfd2ae, 0xffcfd2ae, + 0xffc890e0, 0xffc890e0, 0x0001efca, 0x0001efca, + 0xb35b6f75, 0xb35b6f75, 0xc20bd772, 0xc20bd772, + 0xc4cff072, 0xc4cff072, 0x748f7cf6, 0x748f7cf6, + 0xaa1f5280, 0xaa1f5280, 0x5b2592ae, 0x5b2592ae, + 0x21e490e0, 0x21e490e0, 0x80fb2fca, 0x80fb2fca, + 0x003410f2, 0x003410f2, 0xfff0fe85, 0xfff0fe85, + 0x0020c638, 0x0020c638, 0x00296e9f, 0x00296e9f, + 0xffd2b7a3, 0xffd2b7a3, 0xffc7a44b, 0xffc7a44b, + 0xfff9ba6d, 0xfff9ba6d, 0xffda3409, 0xffda3409, + 0xd15250f2, 0xd15250f2, 0xf1419e85, 0xf1419e85, + 0xdce7c638, 0xdce7c638, 0x5a7d4e9f, 0x5a7d4e9f, + 0x114717a3, 0x114717a3, 0xfad1044b, 0xfad1044b, + 0xb4c75a6d, 0xb4c75a6d, 0x65db5409, 0x65db5409, + 0xfff5c282, 0xfff5c282, 0xffed4113, 0xffed4113, + 0xffffa63b, 0xffffa63b, 0xffec09f7, 0xffec09f7, + 0xfffa2bdd, 0xfffa2bdd, 0x001495d4, 0x001495d4, + 0x001c4563, 0x001c4563, 0xffea2c62, 0xffea2c62, + 0x7f460282, 0x7f460282, 0x6a8fa113, 0x6a8fa113, + 0xc347063b, 0xc347063b, 0x61aae9f7, 0x61aae9f7, + 0xcaf5cbdd, 0xcaf5cbdd, 0xf8cf15d4, 0xf8cf15d4, + 0x6348a563, 0x6348a563, 0x9c766c62, 0x9c766c62, + 0xffccfbe9, 0xffccfbe9, 0x00040af0, 0x00040af0, + 0x0007c417, 0x0007c417, 0x002f4588, 0x002f4588, + 0x0000ad00, 0x0000ad00, 0xffef36be, 0xffef36be, + 0x000dcd44, 0x000dcd44, 0x003c675a, 0x003c675a, + 0x4eca1be9, 0x4eca1be9, 0xc9620af0, 0xc9620af0, + 0x490aa417, 0x490aa417, 0x44e04588, 0x44e04588, + 0x95a0ad00, 0x95a0ad00, 0x7fc6f6be, 0x7fc6f6be, + 0x27b64d44, 0x27b64d44, 0x4827a75a, 0x4827a75a, + 0xffc72bca, 0xffc72bca, 0xffffde7e, 0xffffde7e, + 0x00193948, 0x00193948, 0xffce69c0, 0xffce69c0, + 0x0024756c, 0x0024756c, 0xfffcc7df, 0xfffcc7df, + 0x000b98a1, 0x000b98a1, 0xffebe808, 0xffebe808, + 0x28406bca, 0x28406bca, 0xb4cf9e7e, 0xb4cf9e7e, + 0xa3423948, 0xa3423948, 0xed0669c0, 0xed0669c0, + 0x88d1f56c, 0x88d1f56c, 0x2578a7df, 0x2578a7df, + 0xa69fb8a1, 0xa69fb8a1, 0x98ece808, 0x98ece808, + 0x0002e46c, 0x0002e46c, 0xffc9c808, 0xffc9c808, + 0x003036c2, 0x003036c2, 0xffe3bff6, 0xffe3bff6, + 0xffdb3c93, 0xffdb3c93, 0xfffd4ae0, 0xfffd4ae0, + 0x00141305, 0x00141305, 0x00147792, 0x00147792, + 0xd690646c, 0xd690646c, 0x54cac808, 0x54cac808, + 0xae0876c2, 0xae0876c2, 0x54e27ff6, 0x54e27ff6, + 0x69ed9c93, 0x69ed9c93, 0xb9594ae0, 0xb9594ae0, + 0x13f4b305, 0x13f4b305, 0x0e06b792, 0x0e06b792, + 0x00139e25, 0x00139e25, 0xffe7d0e0, 0xffe7d0e0, + 0xfff39944, 0xfff39944, 0xffea0802, 0xffea0802, + 0xffd1eea2, 0xffd1eea2, 0xffc4c79c, 0xffc4c79c, + 0xffc8a057, 0xffc8a057, 0x003a97d9, 0x003a97d9, + 0xf5583e25, 0xf5583e25, 0x0a03d0e0, 0x0a03d0e0, + 0xe11c1944, 0xe11c1944, 0x47ea4802, 0x47ea4802, + 0x74a62ea2, 0x74a62ea2, 0x3ab8479c, 0x3ab8479c, + 0x44538057, 0x44538057, 0xcab5b7d9, 0xcab5b7d9, + 0x001fea93, 0x0033ff5a, 0x002358d4, 0x003a41f8, + 0xffccff72, 0x00223dfb, 0xffdaab9f, 0xffc9a422, + 0x000412f5, 0x00252587, 0xffed24f0, 0x00359b5d, + 0xffca48a0, 0xffc6a2fc, 0xffedbb56, 0xffcf45de, + 0xfff24a93, 0x3b1f3f5a, 0x513dd8d4, 0x2c7941f8, + 0xaebb3f72, 0x36619dfb, 0x01ce8b9f, 0xab4de422, + 0xdbe2b2f5, 0xfd560587, 0xec8b24f0, 0x79213b5d, + 0x78de48a0, 0x462622fc, 0x64587b56, 0x718b05de, + 0x000dbe5e, 0x001c5e1a, 0x000de0e6, 0x000c7f5a, + 0x00078f83, 0xffe7628a, 0xffff5704, 0xfff806fc, + 0xfff60021, 0xffd05af6, 0x001f0084, 0x0030ef86, + 0xffc9b97d, 0xfff7fcd6, 0xfff44592, 0xffc921c2, + 0x00d97e5e, 0xe6df9e1a, 0xe12aa0e6, 0x4af7bf5a, + 0x3c77ef83, 0xcf38a28a, 0x78dfd704, 0x72d786fc, + 0x337a2021, 0x682f1af6, 0xae2f8084, 0x7321af86, + 0x6c79597d, 0xec92bcd6, 0x07a68592, 0x4b0161c2, + 0x00053919, 0x0004610c, 0xffdacd41, 0x003eb01b, + 0x003472e7, 0xffcd003b, 0x001a7cc7, 0x00031924, + 0x002b5ee5, 0x00291199, 0xffd87a3a, 0x00134d71, + 0x003de11c, 0x00130984, 0x0025f051, 0x00185a46, + 0x7ea85919, 0x3625e10c, 0xbd02ed41, 0x34c2101b, + 0xb71152e7, 0x6e54603b, 0x08335cc7, 0x61279924, + 0x8d87fee5, 0xb9dc3199, 0xda1fba3a, 0x75416d71, + 0x9e61611c, 0xaf438984, 0xd9b01051, 0x00611a46, + 0xffc68518, 0x001314be, 0x00283891, 0xffc9db90, + 0xffd25089, 0x001c853f, 0x001d0b4b, 0xffeff6a6, + 0xffeba8be, 0x0012e11b, 0xffcd5e3e, 0xffea2d2f, + 0xfff91de4, 0x001406c7, 0x00327283, 0xffe20d6e, + 0xa4698518, 0xfbaad4be, 0x02ba5891, 0xb33bdb90, + 0x29637089, 0xed44653f, 0x28066b4b, 0x43c4b6a6, + 0x0e0368be, 0x3ab6411b, 0x84951e3e, 0x6a100d2f, + 0xc1b59de4, 0x396ce6c7, 0x1902d283, 0x428fcd6e, + 0xffec7953, 0x001d4099, 0xffd92578, 0xffeb05ad, + 0x0016e405, 0x000bdbe7, 0x00221de8, 0x0033f8cf, + 0xfff7b934, 0xffd4ca0c, 0xffe67ff8, 0xffe3d157, + 0xffd8911b, 0xffc72c12, 0x000910d8, 0xffc65e1f, + 0x3196d953, 0xbfb06099, 0x48882578, 0x3e20a5ad, + 0xee178405, 0x2408bbe7, 0xefdf1de8, 0x53cdd8cf, + 0x2d1e3934, 0xc3164a0c, 0xb3e57ff8, 0x2a8eb157, + 0xf07bf11b, 0x24496c12, 0x162410d8, 0x380a3e1f, + 0xffe14658, 0x00251d8b, 0x002573b7, 0xfffd7c8f, + 0x001ddd98, 0x00336898, 0x0002d4bb, 0xffed93a7, + 0xffcf6cbe, 0x00027c1c, 0x0018aa08, 0x002dfd71, + 0x000c5ca5, 0x0019379a, 0xffc7a167, 0xffe48c3d, + 0x5cac4658, 0x0a567d8b, 0xaf1c53b7, 0xa40f5c8f, + 0x4fd0dd98, 0x81466898, 0xe91a34bb, 0x7ae273a7, + 0x86672cbe, 0xb185fc1c, 0x3159aa08, 0xcb5c1d71, + 0xcd20fca5, 0xc20c779a, 0xdc748167, 0x66ec2c3d, + 0xffd1a13c, 0x0035c539, 0x003b0115, 0x00041dc0, + 0x0021c4f7, 0xfff11bf4, 0x001a35e7, 0x0007340e, + 0xfff97d45, 0x001a4cd0, 0xffe47cae, 0x001d2668, + 0xffe68e98, 0xffef2633, 0xfffc05da, 0xffc57fdb, + 0x85f9213c, 0x005ce539, 0x29dda115, 0xa3bc1dc0, + 0x9940a4f7, 0xf96f9bf4, 0xef5715e7, 0x1788f40e, + 0xa1221d45, 0x21b44cd0, 0xf07a3cae, 0x10ea2668, + 0xe5b98e98, 0x97358633, 0xfbb745da, 0x2e40dfdb, + 0xffd32764, 0xffdde1af, 0xfff993dd, 0xffdd1d09, + 0x0002cc93, 0xfff11805, 0x00189c2a, 0xffc9e5a9, + 0xfff78a50, 0x003bcf2c, 0xffff434e, 0xffeb36df, + 0x003c15ca, 0x00155e68, 0xfff316b6, 0x001e29ce, + 0x42bfa764, 0xa093c1af, 0xb7f533dd, 0x42fe3d09, + 0x5c152c93, 0x3471b805, 0xa69ddc2a, 0x0bff05a9, + 0x09418a50, 0x94214f2c, 0x7969034e, 0x734716df, + 0xc5f555ca, 0x17e25e68, 0xdfc9d6b6, 0x1657e9ce, +}; + +XALIGNED(16) static const word32 L_mldsa_avx512_zeta1_1p[] WC_X64I_UNUSED = { + 0xffc97e01, 0xffc97e01, 0xffc97e01, 0xffc97e01, + 0xffc97e01, 0xffc97e01, 0xffc97e01, 0xffc97e01, + 0xffc97e01, 0xffc97e01, 0xffc97e01, 0xffc97e01, + 0xffc97e01, 0xffc97e01, 0xffc97e01, 0xffc97e01, +}; + +XALIGNED(16) static const word32 L_mldsa_avx512_half_1p[] WC_X64I_UNUSED = { + 0x00400000, 0x00400000, 0x00400000, 0x00400000, + 0x00400000, 0x00400000, 0x00400000, 0x00400000, + 0x00400000, 0x00400000, 0x00400000, 0x00400000, + 0x00400000, 0x00400000, 0x00400000, 0x00400000, +}; + +XALIGNED(16) static const word32 L_mldsa_avx512_zetas_inv_1p[] + WC_X64I_UNUSED = { + 0xffe1d632, 0x000ce94a, 0xffeaa198, 0xffc3ea36, + 0x0014c921, 0x0000bcb2, 0xffc430d4, 0x000875b0, + 0x00361a57, 0xffe763d6, 0x000ee7fb, 0xfffd336d, + 0x0022e2f7, 0x00066c23, 0x00221e51, 0x002cd89c, + 0xe9a81632, 0x2036294a, 0xe81da198, 0x3a0aaa36, + 0x8cb8e921, 0x8696fcb2, 0x6bdeb0d4, 0xf6be75b0, + 0xf400fa57, 0x596223d6, 0xcb8e47fb, 0xa3ead36d, + 0xbd01c2f7, 0x480acc23, 0x5f6c3e51, 0xbd40589c, + 0x003a8025, 0x0003fa26, 0x0010d9cd, 0x00197168, + 0xffe2d998, 0x001b8352, 0xffe5b330, 0x000682bb, + 0xfff8cbf2, 0xffe5ca19, 0x000ee40c, 0xffde3b09, + 0xfffbe240, 0xffc4feeb, 0xffca3ac7, 0x002e5ec4, + 0xd1bf2025, 0x0448ba26, 0x68ca79cd, 0x1a467168, + 0xef15d998, 0x0f85c352, 0xde4bb330, 0x5edde2bb, + 0xe8770bf2, 0x10a8ea19, 0x0690640c, 0x66bf5b09, + 0x5c43e240, 0xd6225eeb, 0xffa31ac7, 0x7a06dec4, + 0x001b73c3, 0x00385e99, 0xffe6c866, 0xfff3a35b, + 0xffd2028f, 0xffe755f8, 0xfffd83e4, 0x00309342, + 0x00126c59, 0xfffd2b45, 0xffcc9768, 0xffe22268, + 0x00028371, 0xffda8c49, 0xffdae275, 0x001eb9a8, + 0x9913d3c3, 0x238b7e99, 0x3df38866, 0x32df035b, + 0x34a3e28f, 0xcea655f8, 0x4e7a03e4, 0x7998d342, + 0x851d8c59, 0x16e5cb45, 0x7eb99768, 0xb02f2268, + 0x5bf0a371, 0x50e3ac49, 0xf5a98275, 0xa353b9a8, + 0x0039a1e1, 0xfff6ef28, 0x0038d3ee, 0x00276ee5, + 0x001c2ea9, 0x00198008, 0x002b35f4, 0x000846cc, + 0xffcc0731, 0xffdde218, 0xfff42419, 0xffe91bfb, + 0x0014fa53, 0x0026da88, 0xffe2bf67, 0x001386ad, + 0xc7f5c1e1, 0xe9dbef28, 0xdbb693ee, 0x0f840ee5, + 0xd5714ea9, 0x4c1a8008, 0x3ce9b5f4, 0xd2e1c6cc, + 0xac322731, 0x1020e218, 0xdbf74419, 0x11e87bfb, + 0xc1df5a53, 0xb777da88, 0x404f9f67, 0xce6926ad, + 0x001df292, 0xffcd8d7d, 0xffebf939, 0x0006e21c, + 0x0015d2d1, 0x0032a1c2, 0xffed1ee5, 0x00145742, + 0x0010095a, 0xffe2f4b5, 0xffe37ac1, 0x002daf77, + 0x00362470, 0xffd7c76f, 0xffeceb42, 0x00397ae8, + 0xbd703292, 0xe6fd2d7d, 0xc6931939, 0x3e4a621c, + 0x95eff2d1, 0x7b6ae1c2, 0xc549bee5, 0xf1fc9742, + 0xbc3b495a, 0xd7f994b5, 0x12bb9ac1, 0xd69c8f77, + 0x4cc42470, 0xfd45a76f, 0x04552b42, 0x5b967ae8, + 0xffe7a5ba, 0xffda0faf, 0xffecf67c, 0xffc21ee4, + 0xffecb28f, 0x002785c6, 0xffd6ee67, 0xffd4a11b, + 0xfffce6dc, 0xffe58339, 0x0032ffc5, 0xffcb8d19, + 0xffc14fe5, 0x002532bf, 0xfffb9ef4, 0xfffac6e7, + 0xff9ee5ba, 0x264fefaf, 0x50bc767c, 0x619e9ee4, + 0x8abe928f, 0x25e045c6, 0x4623ce67, 0x7278011b, + 0x9ed866dc, 0xf7cca339, 0x91ab9fc5, 0x48eead19, + 0xcb3defe5, 0x42fd12bf, 0xc9da1ef4, 0x8157a6e7, + 0x0036de3e, 0x000bba6e, 0x0008032a, 0x00364683, + 0xffcf107a, 0xffe0ff7c, 0x002fa50a, 0x0009ffdf, + 0x0007f904, 0x0000a8fc, 0x00189d76, 0xfff8707d, + 0xfff380a6, 0xfff21f1a, 0xffe3a1e6, 0xfff241a2, + 0xb4fe9e3e, 0xf8597a6e, 0x136d432a, 0x9386a683, + 0x8cde507a, 0x51d07f7c, 0x97d0e50a, 0xcc85dfdf, + 0x8d287904, 0x872028fc, 0x30c75d76, 0xc388107d, + 0xb50840a6, 0x1ed55f1a, 0x192061e6, 0xff2681a2, + 0x0030ba22, 0x001244aa, 0x00395d04, 0x0035b760, + 0xffca64a3, 0x0012db10, 0xffdada79, 0xfffbed0b, + 0x00365bde, 0x00255461, 0xffddc205, 0x0033008e, + 0xffc5be08, 0xffdca72c, 0xffcc00a6, 0xffe0156d, + 0x8e74fa22, 0x9ba784aa, 0xb9d9dd04, 0x8721b760, + 0x86dec4a3, 0x1374db10, 0x02a9fa79, 0x241d4d0b, + 0x54b21bde, 0xfe317461, 0xc99e6205, 0x5144c08e, + 0xd386be08, 0xaec2272c, 0xc4e0c0a6, 0x000db56d, + 0xffc56827, 0xffc56827, 0x00375fa9, 0x00375fa9, + 0x003b3864, 0x003b3864, 0x002e115e, 0x002e115e, + 0x0015f7fe, 0x0015f7fe, 0x000c66bc, 0x000c66bc, + 0x00182f20, 0x00182f20, 0xffec61db, 0xffec61db, + 0x354a4827, 0x354a4827, 0xbbac7fa9, 0xbbac7fa9, + 0xc547b864, 0xc547b864, 0x8b59d15e, 0x8b59d15e, + 0xb815b7fe, 0xb815b7fe, 0x1ee3e6bc, 0x1ee3e6bc, + 0xf5fc2f20, 0xf5fc2f20, 0x0aa7c1db, 0x0aa7c1db, + 0xffeb886e, 0xffeb886e, 0xffebecfb, 0xffebecfb, + 0x0002b520, 0x0002b520, 0x0024c36d, 0x0024c36d, + 0x001c400a, 0x001c400a, 0xffcfc93e, 0xffcfc93e, + 0x003637f8, 0x003637f8, 0xfffd1b94, 0xfffd1b94, + 0xf1f9486e, 0xf1f9486e, 0xec0b4cfb, 0xec0b4cfb, + 0x46a6b520, 0x46a6b520, 0x9612636d, 0x9612636d, + 0xab1d800a, 0xab1d800a, 0x51f7893e, 0x51f7893e, + 0xab3537f8, 0xab3537f8, 0x296f9b94, 0x296f9b94, + 0x001417f8, 0x001417f8, 0xfff4675f, 0xfff4675f, + 0x00033821, 0x00033821, 0xffdb8a94, 0xffdb8a94, + 0x00319640, 0x00319640, 0xffe6c6b8, 0xffe6c6b8, + 0x00002182, 0x00002182, 0x0038d436, 0x0038d436, + 0x671317f8, 0x671317f8, 0x5960475f, 0x5960475f, + 0xda875821, 0xda875821, 0x772e0a94, 0x772e0a94, + 0x12f99640, 0x12f99640, 0x5cbdc6b8, 0x5cbdc6b8, + 0x4b306182, 0x4b306182, 0xd7bf9436, 0xd7bf9436, + 0xffc398a6, 0xffc398a6, 0xfff232bc, 0xfff232bc, + 0x0010c942, 0x0010c942, 0xffff5300, 0xffff5300, + 0xffd0ba78, 0xffd0ba78, 0xfff83be9, 0xfff83be9, + 0xfffbf510, 0xfffbf510, 0x00330417, 0x00330417, + 0xb7d858a6, 0xb7d858a6, 0xd849b2bc, 0xd849b2bc, + 0x80390942, 0x80390942, 0x6a5f5300, 0x6a5f5300, + 0xbb1fba78, 0xbb1fba78, 0xb6f55be9, 0xb6f55be9, + 0x369df510, 0x369df510, 0xb135e417, 0xb135e417, + 0x0015d39e, 0x0015d39e, 0xffe3ba9d, 0xffe3ba9d, + 0xffeb6a2c, 0xffeb6a2c, 0x0005d423, 0x0005d423, + 0x0013f609, 0x0013f609, 0x000059c5, 0x000059c5, + 0x0012beed, 0x0012beed, 0x000a3d7e, 0x000a3d7e, + 0x6389939e, 0x6389939e, 0x9cb75a9d, 0x9cb75a9d, + 0x0730ea2c, 0x0730ea2c, 0x350a3423, 0x350a3423, + 0x9e551609, 0x9e551609, 0x3cb8f9c5, 0x3cb8f9c5, + 0x95705eed, 0x95705eed, 0x80b9fd7e, 0x80b9fd7e, + 0x0025cbf7, 0x0025cbf7, 0x00064593, 0x00064593, + 0x00385bb5, 0x00385bb5, 0x002d485d, 0x002d485d, + 0xffd69161, 0xffd69161, 0xffdf39c8, 0xffdf39c8, + 0x000f017b, 0x000f017b, 0xffcbef0e, 0xffcbef0e, + 0x9a24abf7, 0x9a24abf7, 0x4b38a593, 0x4b38a593, + 0x052efbb5, 0x052efbb5, 0xeeb8e85d, 0xeeb8e85d, + 0xa582b161, 0xa582b161, 0x231839c8, 0x231839c8, + 0x0ebe617b, 0x0ebe617b, 0x2eadaf0e, 0x2eadaf0e, + 0xfffe1036, 0xfffe1036, 0x00376f20, 0x00376f20, + 0x00302d52, 0x00302d52, 0x0030ad80, 0x0030ad80, + 0x000f430a, 0x000f430a, 0x003e4f8e, 0x003e4f8e, + 0xffe2688e, 0xffe2688e, 0x0013308b, 0x0013308b, + 0x7f04d036, 0x7f04d036, 0xde1b6f20, 0xde1b6f20, + 0xa4da6d52, 0xa4da6d52, 0x55e0ad80, 0x55e0ad80, + 0x8b70830a, 0x8b70830a, 0x3b300f8e, 0x3b300f8e, + 0x3df4288e, 0x3df4288e, 0x4ca4908b, 0x4ca4908b, + 0x00183045, 0x00183045, 0xffdeca39, 0xffdeca39, + 0xffcaf612, 0xffcaf612, 0x001629a3, 0x001629a3, + 0x002e67e7, 0x002e67e7, 0x00381e31, 0x00381e31, + 0x0017537f, 0x0017537f, 0x003bf91b, 0x003bf91b, + 0xf7a0d045, 0xf7a0d045, 0xa0a5ea39, 0xa0a5ea39, + 0x9d8d3612, 0x9d8d3612, 0x7fca89a3, 0x7fca89a3, + 0x75ab47e7, 0x75ab47e7, 0xaf7e3e31, 0xaf7e3e31, + 0x2707337f, 0x2707337f, 0x5ddf591b, 0x5ddf591b, + 0x002ab0d3, 0x002ab0d3, 0x002ab0d3, 0x002ab0d3, + 0xffe062ac, 0xffe062ac, 0xffe062ac, 0xffe062ac, + 0x002703d0, 0x002703d0, 0x002703d0, 0x002703d0, + 0xffc47acc, 0xffc47acc, 0xffc47acc, 0xffc47acc, + 0x38c510d3, 0x38c510d3, 0x38c510d3, 0x38c510d3, + 0x6635e2ac, 0x6635e2ac, 0x6635e2ac, 0x6635e2ac, + 0x38a103d0, 0x38a103d0, 0x38a103d0, 0x38a103d0, + 0x591dfacc, 0x591dfacc, 0x591dfacc, 0x591dfacc, + 0xffc4c7ad, 0xffc4c7ad, 0xffc4c7ad, 0xffc4c7ad, + 0xfff1708a, 0xfff1708a, 0xfff1708a, 0xfff1708a, + 0xfff7e466, 0xfff7e466, 0xfff7e466, 0xfff7e466, + 0xfff39c58, 0xfff39c58, 0xfff39c58, 0xfff39c58, + 0x763a67ad, 0x763a67ad, 0x763a67ad, 0x763a67ad, + 0x1102b08a, 0x1102b08a, 0x1102b08a, 0x1102b08a, + 0x6184a466, 0x6184a466, 0x6184a466, 0x6184a466, + 0xa77e9c58, 0xa77e9c58, 0xa77e9c58, 0xa77e9c58, + 0xffc78c74, 0xffc78c74, 0xffc78c74, 0xffc78c74, + 0x00186ba4, 0x00186ba4, 0x00186ba4, 0x00186ba4, + 0x0020a9e9, 0x0020a9e9, 0x0020a9e9, 0x0020a9e9, + 0xffca7bc1, 0xffca7bc1, 0xffca7bc1, 0xffca7bc1, + 0x87560c74, 0x87560c74, 0x87560c74, 0x87560c74, + 0xcb8ceba4, 0xcb8ceba4, 0xcb8ceba4, 0xcb8ceba4, + 0xc4ddc9e9, 0xc4ddc9e9, 0xc4ddc9e9, 0xc4ddc9e9, + 0x72c29bc1, 0x72c29bc1, 0x72c29bc1, 0x72c29bc1, + 0x003a50a7, 0x003a50a7, 0x003a50a7, 0x003a50a7, + 0xffca81e2, 0xffca81e2, 0xffca81e2, 0xffca81e2, + 0x0019152a, 0x0019152a, 0x0019152a, 0x0019152a, + 0x0019edc3, 0x0019edc3, 0x0019edc3, 0x0019edc3, + 0x92cf30a7, 0x92cf30a7, 0x92cf30a7, 0x92cf30a7, + 0xe706c1e2, 0xe706c1e2, 0xe706c1e2, 0xe706c1e2, + 0x35be552a, 0x35be552a, 0x35be552a, 0x35be552a, + 0x68524dc3, 0x68524dc3, 0x68524dc3, 0x68524dc3, + 0x00083aa3, 0x00083aa3, 0x00083aa3, 0x00083aa3, + 0xffdc2964, 0xffdc2964, 0xffdc2964, 0xffdc2964, + 0x000495b3, 0x000495b3, 0x000495b3, 0x000495b3, + 0xffc9fc00, 0xffc9fc00, 0xffc9fc00, 0xffc9fc00, + 0x41dc9aa3, 0x41dc9aa3, 0x41dc9aa3, 0x41dc9aa3, + 0x6308a964, 0x6308a964, 0x6308a964, 0x6308a964, + 0x853af5b3, 0x853af5b3, 0x853af5b3, 0x853af5b3, + 0x3f49fc00, 0x3f49fc00, 0x3f49fc00, 0x3f49fc00, + 0x002bc1bf, 0x002bc1bf, 0x002bc1bf, 0x002bc1bf, + 0xffc9756a, 0xffc9756a, 0xffc9756a, 0xffc9756a, + 0x002e7184, 0x002e7184, 0x002e7184, 0x002e7184, + 0x003aea7b, 0x003aea7b, 0x003aea7b, 0x003aea7b, + 0x94e3a1bf, 0x94e3a1bf, 0x94e3a1bf, 0x94e3a1bf, + 0x2176b56a, 0x2176b56a, 0x2176b56a, 0x2176b56a, + 0x1c5ef184, 0x1c5ef184, 0x1c5ef184, 0x1c5ef184, + 0x0c0a4a7b, 0x0c0a4a7b, 0x0c0a4a7b, 0x0c0a4a7b, + 0xffc44151, 0xffc44151, 0xffc44151, 0xffc44151, + 0x0026b82c, 0x0026b82c, 0x0026b82c, 0x0026b82c, + 0x0036cfd4, 0x0036cfd4, 0x0036cfd4, 0x0036cfd4, + 0x00195afd, 0x00195afd, 0x00195afd, 0x00195afd, + 0x236e6151, 0x236e6151, 0x236e6151, 0x236e6151, + 0x712c382c, 0x712c382c, 0x712c382c, 0x712c382c, + 0x40314fd4, 0x40314fd4, 0x40314fd4, 0x40314fd4, + 0xa0f8fafd, 0xa0f8fafd, 0xa0f8fafd, 0xa0f8fafd, + 0xffca213b, 0xffca213b, 0xffca213b, 0xffca213b, + 0xffd10b33, 0xffd10b33, 0xffd10b33, 0xffd10b33, + 0xfffe89e0, 0xfffe89e0, 0xfffe89e0, 0xfffe89e0, + 0xffd6b599, 0xffd6b599, 0xffd6b599, 0xffd6b599, + 0x9271813b, 0x9271813b, 0x9271813b, 0x9271813b, + 0x53b76b33, 0x53b76b33, 0x53b76b33, 0x53b76b33, + 0x613a89e0, 0x613a89e0, 0x613a89e0, 0x613a89e0, + 0x6e09d599, 0x6e09d599, 0x6e09d599, 0x6e09d599, + 0xfffbba3b, 0xfffbba3b, 0xfffbba3b, 0xfffbba3b, + 0xfffbba3b, 0xfffbba3b, 0xfffbba3b, 0xfffbba3b, + 0xffc2cdff, 0xffc2cdff, 0xffc2cdff, 0xffc2cdff, + 0xffc2cdff, 0xffc2cdff, 0xffc2cdff, 0xffc2cdff, + 0x45c31a3b, 0x45c31a3b, 0x45c31a3b, 0x45c31a3b, + 0x45c31a3b, 0x45c31a3b, 0x45c31a3b, 0x45c31a3b, + 0x5602adff, 0x5602adff, 0x5602adff, 0x5602adff, + 0x5602adff, 0x5602adff, 0x5602adff, 0x5602adff, + 0x00004bde, 0x00004bde, 0x00004bde, 0x00004bde, + 0x00004bde, 0x00004bde, 0x00004bde, 0x00004bde, + 0xffe52fcb, 0xffe52fcb, 0xffe52fcb, 0xffe52fcb, + 0xffe52fcb, 0xffe52fcb, 0xffe52fcb, 0xffe52fcb, + 0x927c0bde, 0x927c0bde, 0x927c0bde, 0x927c0bde, + 0x927c0bde, 0x927c0bde, 0x927c0bde, 0x927c0bde, + 0xec5e8fcb, 0xec5e8fcb, 0xec5e8fcb, 0xec5e8fcb, + 0xec5e8fcb, 0xec5e8fcb, 0xec5e8fcb, 0xec5e8fcb, + 0x00320368, 0x00320368, 0x00320368, 0x00320368, + 0x00320368, 0x00320368, 0x00320368, 0x00320368, + 0x00155b09, 0x00155b09, 0x00155b09, 0x00155b09, + 0x00155b09, 0x00155b09, 0x00155b09, 0x00155b09, + 0x2c9f0368, 0x2c9f0368, 0x2c9f0368, 0x2c9f0368, + 0x2c9f0368, 0x2c9f0368, 0x2c9f0368, 0x2c9f0368, + 0x4af67b09, 0x4af67b09, 0x4af67b09, 0x4af67b09, + 0x4af67b09, 0x4af67b09, 0x4af67b09, 0x4af67b09, + 0x003ae519, 0x003ae519, 0x003ae519, 0x003ae519, + 0x003ae519, 0x003ae519, 0x003ae519, 0x003ae519, + 0x0020522a, 0x0020522a, 0x0020522a, 0x0020522a, + 0x0020522a, 0x0020522a, 0x0020522a, 0x0020522a, + 0x345e0519, 0x345e0519, 0x345e0519, 0x345e0519, + 0x345e0519, 0x345e0519, 0x345e0519, 0x345e0519, + 0x9d65922a, 0x9d65922a, 0x9d65922a, 0x9d65922a, + 0x9d65922a, 0x9d65922a, 0x9d65922a, 0x9d65922a, + 0x00202c85, 0x00202c85, 0x00202c85, 0x00202c85, + 0x00202c85, 0x00202c85, 0x00202c85, 0x00202c85, + 0xffd80698, 0xffd80698, 0xffd80698, 0xffd80698, + 0xffd80698, 0xffd80698, 0xffd80698, 0xffd80698, + 0xd730cc85, 0xd730cc85, 0xd730cc85, 0xd730cc85, + 0xd730cc85, 0xd730cc85, 0xd730cc85, 0xd730cc85, + 0x14ab0698, 0x14ab0698, 0x14ab0698, 0x14ab0698, + 0x14ab0698, 0x14ab0698, 0x14ab0698, 0x14ab0698, + 0x00111560, 0x00111560, 0x00111560, 0x00111560, + 0x00111560, 0x00111560, 0x00111560, 0x00111560, + 0x00086270, 0x00086270, 0x00086270, 0x00086270, + 0x00086270, 0x00086270, 0x00086270, 0x00086270, + 0xf2bd1560, 0xf2bd1560, 0xf2bd1560, 0xf2bd1560, + 0xf2bd1560, 0xf2bd1560, 0xf2bd1560, 0xf2bd1560, + 0x94566270, 0x94566270, 0x94566270, 0x94566270, + 0x94566270, 0x94566270, 0x94566270, 0x94566270, + 0xffc94878, 0xffc94878, 0xffc94878, 0xffc94878, + 0xffc94878, 0xffc94878, 0xffc94878, 0xffc94878, + 0x00107a5c, 0x00107a5c, 0x00107a5c, 0x00107a5c, + 0x00107a5c, 0x00107a5c, 0x00107a5c, 0x00107a5c, + 0xccd84878, 0xccd84878, 0xccd84878, 0xccd84878, + 0xccd84878, 0xccd84878, 0xccd84878, 0xccd84878, + 0x515bfa5c, 0x515bfa5c, 0x515bfa5c, 0x515bfa5c, + 0x515bfa5c, 0x515bfa5c, 0x515bfa5c, 0x515bfa5c, + 0xfff05f90, 0xfff05f90, 0xfff05f90, 0xfff05f90, + 0xfff05f90, 0xfff05f90, 0xfff05f90, 0xfff05f90, + 0xffd669a8, 0xffd669a8, 0xffd669a8, 0xffd669a8, + 0xffd669a8, 0xffd669a8, 0xffd669a8, 0xffd669a8, + 0x83e25f90, 0x83e25f90, 0x83e25f90, 0x83e25f90, + 0x83e25f90, 0x83e25f90, 0x83e25f90, 0x83e25f90, + 0x990b69a8, 0x990b69a8, 0x990b69a8, 0x990b69a8, + 0x990b69a8, 0x990b69a8, 0x990b69a8, 0x990b69a8, + 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, + 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, + 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, + 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, 0xffd71ad9, + 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, + 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, + 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, + 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, 0xdab23ad9, + 0xffd085b7, 0xffd085b7, 0xffd085b7, 0xffd085b7, + 0xffd085b7, 0xffd085b7, 0xffd085b7, 0xffd085b7, + 0xffd085b7, 0xffd085b7, 0xffd085b7, 0xffd085b7, + 0xffd085b7, 0xffd085b7, 0xffd085b7, 0xffd085b7, + 0x110765b7, 0x110765b7, 0x110765b7, 0x110765b7, + 0x110765b7, 0x110765b7, 0x110765b7, 0x110765b7, + 0x110765b7, 0x110765b7, 0x110765b7, 0x110765b7, + 0x110765b7, 0x110765b7, 0x110765b7, 0x110765b7, + 0x002c04f7, 0x002c04f7, 0x002c04f7, 0x002c04f7, + 0x002c04f7, 0x002c04f7, 0x002c04f7, 0x002c04f7, + 0x002c04f7, 0x002c04f7, 0x002c04f7, 0x002c04f7, + 0x002c04f7, 0x002c04f7, 0x002c04f7, 0x002c04f7, + 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, + 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, + 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, + 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, 0xe14ae4f7, + 0xffd0658b, 0xffd0658b, 0xffd0658b, 0xffd0658b, + 0xffd0658b, 0xffd0658b, 0xffd0658b, 0xffd0658b, + 0xffd0658b, 0xffd0658b, 0xffd0658b, 0xffd0658b, + 0xffd0658b, 0xffd0658b, 0xffd0658b, 0xffd0658b, + 0x7301c58b, 0x7301c58b, 0x7301c58b, 0x7301c58b, + 0x7301c58b, 0x7301c58b, 0x7301c58b, 0x7301c58b, + 0x7301c58b, 0x7301c58b, 0x7301c58b, 0x7301c58b, + 0x7301c58b, 0x7301c58b, 0x7301c58b, 0x7301c58b, + 0x001feb81, 0x001feb81, 0x001feb81, 0x001feb81, + 0x001feb81, 0x001feb81, 0x001feb81, 0x001feb81, + 0x001feb81, 0x001feb81, 0x001feb81, 0x001feb81, + 0x001feb81, 0x001feb81, 0x001feb81, 0x001feb81, + 0x41100b81, 0x41100b81, 0x41100b81, 0x41100b81, + 0x41100b81, 0x41100b81, 0x41100b81, 0x41100b81, + 0x41100b81, 0x41100b81, 0x41100b81, 0x41100b81, + 0x41100b81, 0x41100b81, 0x41100b81, 0x41100b81, + 0x00057b53, 0x00057b53, 0x00057b53, 0x00057b53, + 0x00057b53, 0x00057b53, 0x00057b53, 0x00057b53, + 0x00057b53, 0x00057b53, 0x00057b53, 0x00057b53, + 0x00057b53, 0x00057b53, 0x00057b53, 0x00057b53, + 0x51efdb53, 0x51efdb53, 0x51efdb53, 0x51efdb53, + 0x51efdb53, 0x51efdb53, 0x51efdb53, 0x51efdb53, + 0x51efdb53, 0x51efdb53, 0x51efdb53, 0x51efdb53, + 0x51efdb53, 0x51efdb53, 0x51efdb53, 0x51efdb53, + 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, + 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, + 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, + 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, 0xffdc16d5, + 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, + 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, + 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, + 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, 0x6c36b6d5, + 0xffe421d5, 0xffe421d5, 0xffe421d5, 0xffe421d5, + 0xffe421d5, 0xffe421d5, 0xffe421d5, 0xffe421d5, + 0xffe421d5, 0xffe421d5, 0xffe421d5, 0xffe421d5, + 0xffe421d5, 0xffe421d5, 0xffe421d5, 0xffe421d5, + 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, + 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, + 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, + 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, 0xed9ec1d5, + 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, + 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, + 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, + 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, 0xfff8e1dc, + 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, + 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, + 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, + 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, 0x9e3461dc, + 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, + 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, + 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, + 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, 0x000d5ed8, + 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, + 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, + 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, + 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, 0x9fe85ed8, + 0x000bdee8, 0x000bdee8, 0x000bdee8, 0x000bdee8, + 0x000bdee8, 0x000bdee8, 0x000bdee8, 0x000bdee8, + 0x000bdee8, 0x000bdee8, 0x000bdee8, 0x000bdee8, + 0x000bdee8, 0x000bdee8, 0x000bdee8, 0x000bdee8, + 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, + 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, + 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, + 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, 0xa7e8dee8, + 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, + 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, + 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, + 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, 0xfffc61bc, + 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, + 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, + 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, + 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, 0x9e33e1bc, + 0x0007eafd, 0x0007eafd, 0x0007eafd, 0x0007eafd, + 0x0007eafd, 0x0007eafd, 0x0007eafd, 0x0007eafd, + 0x0007eafd, 0x0007eafd, 0x0007eafd, 0x0007eafd, + 0x0007eafd, 0x0007eafd, 0x0007eafd, 0x0007eafd, + 0x72e78afd, 0x72e78afd, 0x72e78afd, 0x72e78afd, + 0x72e78afd, 0x72e78afd, 0x72e78afd, 0x72e78afd, + 0x72e78afd, 0x72e78afd, 0x72e78afd, 0x72e78afd, + 0x72e78afd, 0x72e78afd, 0x72e78afd, 0x72e78afd, + 0x0027cefe, 0x0027cefe, 0x0027cefe, 0x0027cefe, + 0x0027cefe, 0x0027cefe, 0x0027cefe, 0x0027cefe, + 0x0027cefe, 0x0027cefe, 0x0027cefe, 0x0027cefe, + 0x0027cefe, 0x0027cefe, 0x0027cefe, 0x0027cefe, + 0x73078efe, 0x73078efe, 0x73078efe, 0x73078efe, + 0x73078efe, 0x73078efe, 0x73078efe, 0x73078efe, + 0x73078efe, 0x73078efe, 0x73078efe, 0x73078efe, + 0x73078efe, 0x73078efe, 0x73078efe, 0x73078efe, + 0xffff9b09, 0xffff9b09, 0xffff9b09, 0xffff9b09, + 0xffff9b09, 0xffff9b09, 0xffff9b09, 0xffff9b09, + 0xffff9b09, 0xffff9b09, 0xffff9b09, 0xffff9b09, + 0xffff9b09, 0xffff9b09, 0xffff9b09, 0xffff9b09, + 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, + 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, + 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, + 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, 0x92e0bb09, + 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, + 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, + 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, + 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, 0x0000a3fa, + 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, + 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, + 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, + 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, 0xff7fe3fa, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_ntt_1p_avx512(sword32* r) +{ + word64 rdi, rax, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z21, z22, z23, z24, z25, z26; + __mmask16 k1, k2; + + rdi = (word64)(size_t)r; + + rax = (word64)(0x7fe001); + z23 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z23 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z23)); + rax = (word32)(0xaaaa); + k1 = (__mmask16)(word32)rax; + rax = (word32)(0x5555); + k2 = (__mmask16)(word32)rax; + z25 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm20, 0)); + z26 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm31, 0)); + /* ntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx512_zetas_1p); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + /* len = 128 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z16 = _mm512_mullo_epi32(z8, z22); + z17 = _mm512_shuffle_epi32(z8, 0xf5); + z8 = _mm512_mul_epi32(z8, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z8, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z8 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z16 = _mm512_mullo_epi32(z10, z22); + z17 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z10, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z10 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z3, z16); + z3 = _mm512_add_epi32(z3, z16); + z16 = _mm512_mullo_epi32(z12, z22); + z17 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z12, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z12 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z5, z16); + z5 = _mm512_add_epi32(z5, z16); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z7, z16); + z7 = _mm512_add_epi32(z7, z16); + /* len = 64 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z16 = _mm512_mullo_epi32(z4, z22); + z17 = _mm512_shuffle_epi32(z4, 0xf5); + z4 = _mm512_mul_epi32(z4, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z4, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z4 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z16 = _mm512_mullo_epi32(z6, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z6 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z3, z16); + z3 = _mm512_add_epi32(z3, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z16 = _mm512_mullo_epi32(z12, z22); + z17 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z12, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z12 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z9, z16); + z9 = _mm512_add_epi32(z9, z16); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z11, z16); + z11 = _mm512_add_epi32(z11, z16); + /* len = 32 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z16 = _mm512_mullo_epi32(z2, z22); + z17 = _mm512_shuffle_epi32(z2, 0xf5); + z2 = _mm512_mul_epi32(z2, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z2, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z2 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z16 = _mm512_mullo_epi32(z6, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z6 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z5, z16); + z5 = _mm512_add_epi32(z5, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z16 = _mm512_mullo_epi32(z10, z22); + z17 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z10, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z10 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z9, z16); + z9 = _mm512_add_epi32(z9, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z13, z16); + z13 = _mm512_add_epi32(z13, z16); + /* len = 16 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1024)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1088)); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1152)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1216)); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1280)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1344)); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1408)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1472)); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1536)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1600)); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1664)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1728)); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1792)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1856)); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 8 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1920)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1984)); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2048)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2112)); + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2176)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2240)); + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2304)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2368)); + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2432)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2496)); + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2560)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2624)); + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2688)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2752)); + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2816)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2880)); + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 4 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2944)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3008)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z0, z24, z1); + z1 = z24; + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3072)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3136)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z2, z24, z3); + z3 = z24; + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3200)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3264)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z4, z24, z5); + z5 = z24; + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3328)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3392)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z6, z24, z7); + z7 = z24; + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3456)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3520)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z8, z16, z9); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z9 = z24; + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3584)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3648)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z10, z16, z11); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z10, z24, z11); + z11 = z24; + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3712)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3776)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z12, z16, z13); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z12, z24, z13); + z13 = z24; + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3840)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3904)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z14, z16, z15); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z14, z24, z15); + z15 = z24; + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 2 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3968)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4032)); + z16 = _mm512_unpacklo_epi64(z0, z1); + z1 = _mm512_unpackhi_epi64(z0, z1); + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4096)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4160)); + z16 = _mm512_unpacklo_epi64(z2, z3); + z3 = _mm512_unpackhi_epi64(z2, z3); + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4224)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4288)); + z16 = _mm512_unpacklo_epi64(z4, z5); + z5 = _mm512_unpackhi_epi64(z4, z5); + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4352)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4416)); + z16 = _mm512_unpacklo_epi64(z6, z7); + z7 = _mm512_unpackhi_epi64(z6, z7); + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4480)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4544)); + z16 = _mm512_unpacklo_epi64(z8, z9); + z9 = _mm512_unpackhi_epi64(z8, z9); + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4608)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4672)); + z16 = _mm512_unpacklo_epi64(z10, z11); + z11 = _mm512_unpackhi_epi64(z10, z11); + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4736)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4800)); + z16 = _mm512_unpacklo_epi64(z12, z13); + z13 = _mm512_unpackhi_epi64(z12, z13); + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4864)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4928)); + z16 = _mm512_unpacklo_epi64(z14, z15); + z15 = _mm512_unpackhi_epi64(z14, z15); + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 1 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4992)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5056)); + z16 = _mm512_slli_epi64(z1, 32); + z17 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z16); + z1 = _mm512_mask_blend_epi32(k2, z1, z17); + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5120)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5184)); + z16 = _mm512_slli_epi64(z3, 32); + z17 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z16); + z3 = _mm512_mask_blend_epi32(k2, z3, z17); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5248)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5312)); + z16 = _mm512_slli_epi64(z5, 32); + z17 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z16); + z5 = _mm512_mask_blend_epi32(k2, z5, z17); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5376)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5440)); + z16 = _mm512_slli_epi64(z7, 32); + z17 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z16); + z7 = _mm512_mask_blend_epi32(k2, z7, z17); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5504)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5568)); + z16 = _mm512_slli_epi64(z9, 32); + z17 = _mm512_srli_epi64(z8, 32); + z8 = _mm512_mask_blend_epi32(k1, z8, z16); + z9 = _mm512_mask_blend_epi32(k2, z9, z17); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5632)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5696)); + z16 = _mm512_slli_epi64(z11, 32); + z17 = _mm512_srli_epi64(z10, 32); + z10 = _mm512_mask_blend_epi32(k1, z10, z16); + z11 = _mm512_mask_blend_epi32(k2, z11, z17); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5760)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5824)); + z16 = _mm512_slli_epi64(z13, 32); + z17 = _mm512_srli_epi64(z12, 32); + z12 = _mm512_mask_blend_epi32(k1, z12, z16); + z13 = _mm512_mask_blend_epi32(k2, z13, z17); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5888)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5952)); + z16 = _mm512_slli_epi64(z15, 32); + z17 = _mm512_srli_epi64(z14, 32); + z14 = _mm512_mask_blend_epi32(k1, z14, z16); + z15 = _mm512_mask_blend_epi32(k2, z15, z17); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_ntt_small_1p_avx512(sword32* r) +{ + word64 rdi, rax, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z21, z22, z23, z24, z25, z26; + __mmask16 k1, k2; + + rdi = (word64)(size_t)r; + + rax = (word64)(0x7fe001); + z23 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z23 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z23)); + rax = (word32)(0xaaaa); + k1 = (__mmask16)(word32)rax; + rax = (word32)(0x5555); + k2 = (__mmask16)(word32)rax; + z25 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm20, 0)); + z26 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm31, 0)); + /* ntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx512_zetas_1p); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + /* len = 128 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_zeta1_1p, 0)); + z22 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_half_1p, 0)); + z16 = _mm512_mullo_epi32(z8, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z8 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z9, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z9 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z16 = _mm512_mullo_epi32(z10, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z10 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z16 = _mm512_mullo_epi32(z11, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z11 = _mm512_sub_epi32(z3, z16); + z3 = _mm512_add_epi32(z3, z16); + z16 = _mm512_mullo_epi32(z12, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z12 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z16 = _mm512_mullo_epi32(z13, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z13 = _mm512_sub_epi32(z5, z16); + z5 = _mm512_add_epi32(z5, z16); + z16 = _mm512_mullo_epi32(z14, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z14 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z16 = _mm512_mullo_epi32(z15, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z15 = _mm512_sub_epi32(z7, z16); + z7 = _mm512_add_epi32(z7, z16); + /* len = 64 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z16 = _mm512_mullo_epi32(z4, z22); + z17 = _mm512_shuffle_epi32(z4, 0xf5); + z4 = _mm512_mul_epi32(z4, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z4, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z4 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z16 = _mm512_mullo_epi32(z6, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z6 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z3, z16); + z3 = _mm512_add_epi32(z3, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z16 = _mm512_mullo_epi32(z12, z22); + z17 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z12, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z12 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z9, z16); + z9 = _mm512_add_epi32(z9, z16); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z11, z16); + z11 = _mm512_add_epi32(z11, z16); + /* len = 32 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z16 = _mm512_mullo_epi32(z2, z22); + z17 = _mm512_shuffle_epi32(z2, 0xf5); + z2 = _mm512_mul_epi32(z2, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z2, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z2 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z16 = _mm512_mullo_epi32(z6, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z6 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z5, z16); + z5 = _mm512_add_epi32(z5, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z16 = _mm512_mullo_epi32(z10, z22); + z17 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z10, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z10 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z9, z16); + z9 = _mm512_add_epi32(z9, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z13, z16); + z13 = _mm512_add_epi32(z13, z16); + /* len = 16 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1024)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1088)); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1152)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1216)); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1280)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1344)); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1408)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1472)); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1536)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1600)); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1664)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1728)); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1792)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1856)); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 8 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1920)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1984)); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2048)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2112)); + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2176)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2240)); + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2304)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2368)); + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2432)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2496)); + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2560)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2624)); + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2688)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2752)); + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2816)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2880)); + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 4 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2944)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3008)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z0, z24, z1); + z1 = z24; + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3072)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3136)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z2, z24, z3); + z3 = z24; + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3200)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3264)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z4, z24, z5); + z5 = z24; + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3328)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3392)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z6, z24, z7); + z7 = z24; + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3456)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3520)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z8, z16, z9); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z9 = z24; + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3584)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3648)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z10, z16, z11); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z10, z24, z11); + z11 = z24; + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3712)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3776)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z12, z16, z13); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z12, z24, z13); + z13 = z24; + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3840)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3904)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z14, z16, z15); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z14, z24, z15); + z15 = z24; + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 2 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3968)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4032)); + z16 = _mm512_unpacklo_epi64(z0, z1); + z1 = _mm512_unpackhi_epi64(z0, z1); + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4096)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4160)); + z16 = _mm512_unpacklo_epi64(z2, z3); + z3 = _mm512_unpackhi_epi64(z2, z3); + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4224)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4288)); + z16 = _mm512_unpacklo_epi64(z4, z5); + z5 = _mm512_unpackhi_epi64(z4, z5); + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4352)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4416)); + z16 = _mm512_unpacklo_epi64(z6, z7); + z7 = _mm512_unpackhi_epi64(z6, z7); + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4480)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4544)); + z16 = _mm512_unpacklo_epi64(z8, z9); + z9 = _mm512_unpackhi_epi64(z8, z9); + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4608)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4672)); + z16 = _mm512_unpacklo_epi64(z10, z11); + z11 = _mm512_unpackhi_epi64(z10, z11); + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4736)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4800)); + z16 = _mm512_unpacklo_epi64(z12, z13); + z13 = _mm512_unpackhi_epi64(z12, z13); + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4864)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4928)); + z16 = _mm512_unpacklo_epi64(z14, z15); + z15 = _mm512_unpackhi_epi64(z14, z15); + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 1 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4992)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5056)); + z16 = _mm512_slli_epi64(z1, 32); + z17 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z16); + z1 = _mm512_mask_blend_epi32(k2, z1, z17); + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5120)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5184)); + z16 = _mm512_slli_epi64(z3, 32); + z17 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z16); + z3 = _mm512_mask_blend_epi32(k2, z3, z17); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5248)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5312)); + z16 = _mm512_slli_epi64(z5, 32); + z17 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z16); + z5 = _mm512_mask_blend_epi32(k2, z5, z17); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5376)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5440)); + z16 = _mm512_slli_epi64(z7, 32); + z17 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z16); + z7 = _mm512_mask_blend_epi32(k2, z7, z17); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5504)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5568)); + z16 = _mm512_slli_epi64(z9, 32); + z17 = _mm512_srli_epi64(z8, 32); + z8 = _mm512_mask_blend_epi32(k1, z8, z16); + z9 = _mm512_mask_blend_epi32(k2, z9, z17); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5632)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5696)); + z16 = _mm512_slli_epi64(z11, 32); + z17 = _mm512_srli_epi64(z10, 32); + z10 = _mm512_mask_blend_epi32(k1, z10, z16); + z11 = _mm512_mask_blend_epi32(k2, z11, z17); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5760)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5824)); + z16 = _mm512_slli_epi64(z13, 32); + z17 = _mm512_srli_epi64(z12, 32); + z12 = _mm512_mask_blend_epi32(k1, z12, z16); + z13 = _mm512_mask_blend_epi32(k2, z13, z17); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5888)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5952)); + z16 = _mm512_slli_epi64(z15, 32); + z17 = _mm512_srli_epi64(z14, 32); + z14 = _mm512_mask_blend_epi32(k1, z14, z16); + z15 = _mm512_mask_blend_epi32(k2, z15, z17); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_ntt_full_1p_avx512(sword32* r) +{ + word64 rdi, rax, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z21, z22, z23, z24, z25, z26, z27, z28; + __mmask16 k1, k2; + + rdi = (word64)(size_t)r; + + rax = (word64)(0x7fe001); + z23 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z23 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z23)); + rax = (word32)(0xaaaa); + k1 = (__mmask16)(word32)rax; + rax = (word32)(0x5555); + k2 = (__mmask16)(word32)rax; + z25 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm20, 0)); + z26 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm31, 0)); + z27 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_permn0, 0)); + z28 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_permn1, 0)); + /* ntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx512_zetas_1p); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + /* len = 128 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z16 = _mm512_mullo_epi32(z8, z22); + z17 = _mm512_shuffle_epi32(z8, 0xf5); + z8 = _mm512_mul_epi32(z8, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z8, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z8 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z16 = _mm512_mullo_epi32(z10, z22); + z17 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z10, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z10 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z3, z16); + z3 = _mm512_add_epi32(z3, z16); + z16 = _mm512_mullo_epi32(z12, z22); + z17 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z12, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z12 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z5, z16); + z5 = _mm512_add_epi32(z5, z16); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z7, z16); + z7 = _mm512_add_epi32(z7, z16); + /* len = 64 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z16 = _mm512_mullo_epi32(z4, z22); + z17 = _mm512_shuffle_epi32(z4, 0xf5); + z4 = _mm512_mul_epi32(z4, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z4, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z4 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z16 = _mm512_mullo_epi32(z6, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z6 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z3, z16); + z3 = _mm512_add_epi32(z3, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z16 = _mm512_mullo_epi32(z12, z22); + z17 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z12, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z12 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z9, z16); + z9 = _mm512_add_epi32(z9, z16); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z11, z16); + z11 = _mm512_add_epi32(z11, z16); + /* len = 32 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z16 = _mm512_mullo_epi32(z2, z22); + z17 = _mm512_shuffle_epi32(z2, 0xf5); + z2 = _mm512_mul_epi32(z2, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z2, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z2 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z16 = _mm512_mullo_epi32(z6, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z6 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z5, z16); + z5 = _mm512_add_epi32(z5, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z16 = _mm512_mullo_epi32(z10, z22); + z17 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z10, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z10 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z9, z16); + z9 = _mm512_add_epi32(z9, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z13, z16); + z13 = _mm512_add_epi32(z13, z16); + /* len = 16 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1024)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1088)); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1152)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1216)); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1280)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1344)); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1408)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1472)); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1536)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1600)); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1664)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1728)); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1792)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1856)); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 8 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1920)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1984)); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2048)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2112)); + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2176)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2240)); + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2304)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2368)); + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2432)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2496)); + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2560)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2624)); + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2688)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2752)); + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2816)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2880)); + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 4 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2944)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3008)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z0, z24, z1); + z1 = z24; + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3072)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3136)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z2, z24, z3); + z3 = z24; + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3200)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3264)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z4, z24, z5); + z5 = z24; + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3328)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3392)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z6, z24, z7); + z7 = z24; + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3456)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3520)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z8, z16, z9); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z9 = z24; + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3584)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3648)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z10, z16, z11); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z10, z24, z11); + z11 = z24; + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3712)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3776)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z12, z16, z13); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z12, z24, z13); + z13 = z24; + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3840)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3904)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z14, z16, z15); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z14, z24, z15); + z15 = z24; + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 2 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3968)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4032)); + z16 = _mm512_unpacklo_epi64(z0, z1); + z1 = _mm512_unpackhi_epi64(z0, z1); + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4096)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4160)); + z16 = _mm512_unpacklo_epi64(z2, z3); + z3 = _mm512_unpackhi_epi64(z2, z3); + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4224)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4288)); + z16 = _mm512_unpacklo_epi64(z4, z5); + z5 = _mm512_unpackhi_epi64(z4, z5); + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4352)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4416)); + z16 = _mm512_unpacklo_epi64(z6, z7); + z7 = _mm512_unpackhi_epi64(z6, z7); + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4480)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4544)); + z16 = _mm512_unpacklo_epi64(z8, z9); + z9 = _mm512_unpackhi_epi64(z8, z9); + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4608)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4672)); + z16 = _mm512_unpacklo_epi64(z10, z11); + z11 = _mm512_unpackhi_epi64(z10, z11); + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4736)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4800)); + z16 = _mm512_unpacklo_epi64(z12, z13); + z13 = _mm512_unpackhi_epi64(z12, z13); + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4864)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4928)); + z16 = _mm512_unpacklo_epi64(z14, z15); + z15 = _mm512_unpackhi_epi64(z14, z15); + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 1 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4992)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5056)); + z16 = _mm512_slli_epi64(z1, 32); + z17 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z16); + z1 = _mm512_mask_blend_epi32(k2, z1, z17); + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5120)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5184)); + z16 = _mm512_slli_epi64(z3, 32); + z17 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z16); + z3 = _mm512_mask_blend_epi32(k2, z3, z17); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5248)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5312)); + z16 = _mm512_slli_epi64(z5, 32); + z17 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z16); + z5 = _mm512_mask_blend_epi32(k2, z5, z17); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5376)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5440)); + z16 = _mm512_slli_epi64(z7, 32); + z17 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z16); + z7 = _mm512_mask_blend_epi32(k2, z7, z17); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5504)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5568)); + z16 = _mm512_slli_epi64(z9, 32); + z17 = _mm512_srli_epi64(z8, 32); + z8 = _mm512_mask_blend_epi32(k1, z8, z16); + z9 = _mm512_mask_blend_epi32(k2, z9, z17); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5632)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5696)); + z16 = _mm512_slli_epi64(z11, 32); + z17 = _mm512_srli_epi64(z10, 32); + z10 = _mm512_mask_blend_epi32(k1, z10, z16); + z11 = _mm512_mask_blend_epi32(k2, z11, z17); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5760)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5824)); + z16 = _mm512_slli_epi64(z13, 32); + z17 = _mm512_srli_epi64(z12, 32); + z12 = _mm512_mask_blend_epi32(k1, z12, z16); + z13 = _mm512_mask_blend_epi32(k2, z13, z17); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5888)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5952)); + z16 = _mm512_slli_epi64(z15, 32); + z17 = _mm512_srli_epi64(z14, 32); + z14 = _mm512_mask_blend_epi32(k1, z14, z16); + z15 = _mm512_mask_blend_epi32(k2, z15, z17); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + z16 = _mm512_unpacklo_epi32(z0, z1); + z17 = _mm512_unpackhi_epi32(z0, z1); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z1 = z28; + z1 = _mm512_permutex2var_epi64(z16, z1, z17); + z0 = z18; + z16 = _mm512_unpacklo_epi32(z2, z3); + z17 = _mm512_unpackhi_epi32(z2, z3); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z3 = z28; + z3 = _mm512_permutex2var_epi64(z16, z3, z17); + z2 = z18; + z16 = _mm512_unpacklo_epi32(z4, z5); + z17 = _mm512_unpackhi_epi32(z4, z5); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z5 = z28; + z5 = _mm512_permutex2var_epi64(z16, z5, z17); + z4 = z18; + z16 = _mm512_unpacklo_epi32(z6, z7); + z17 = _mm512_unpackhi_epi32(z6, z7); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z7 = z28; + z7 = _mm512_permutex2var_epi64(z16, z7, z17); + z6 = z18; + z16 = _mm512_unpacklo_epi32(z8, z9); + z17 = _mm512_unpackhi_epi32(z8, z9); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z9 = z28; + z9 = _mm512_permutex2var_epi64(z16, z9, z17); + z8 = z18; + z16 = _mm512_unpacklo_epi32(z10, z11); + z17 = _mm512_unpackhi_epi32(z10, z11); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z11 = z28; + z11 = _mm512_permutex2var_epi64(z16, z11, z17); + z10 = z18; + z16 = _mm512_unpacklo_epi32(z12, z13); + z17 = _mm512_unpackhi_epi32(z12, z13); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z13 = z28; + z13 = _mm512_permutex2var_epi64(z16, z13, z17); + z12 = z18; + z16 = _mm512_unpacklo_epi32(z14, z15); + z17 = _mm512_unpackhi_epi32(z14, z15); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z15 = z28; + z15 = _mm512_permutex2var_epi64(z16, z15, z17); + z14 = z18; + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_ntt_small_full_1p_avx512(sword32* r) +{ + word64 rdi, rax, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z21, z22, z23, z24, z25, z26, z27, z28; + __mmask16 k1, k2; + + rdi = (word64)(size_t)r; + + rax = (word64)(0x7fe001); + z23 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z23 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z23)); + rax = (word32)(0xaaaa); + k1 = (__mmask16)(word32)rax; + rax = (word32)(0x5555); + k2 = (__mmask16)(word32)rax; + z25 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm20, 0)); + z26 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm31, 0)); + z27 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_permn0, 0)); + z28 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_permn1, 0)); + /* ntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx512_zetas_1p); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + /* len = 128 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_zeta1_1p, 0)); + z22 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_half_1p, 0)); + z16 = _mm512_mullo_epi32(z8, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z8 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z9, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z9 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z16 = _mm512_mullo_epi32(z10, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z10 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z16 = _mm512_mullo_epi32(z11, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z11 = _mm512_sub_epi32(z3, z16); + z3 = _mm512_add_epi32(z3, z16); + z16 = _mm512_mullo_epi32(z12, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z12 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z16 = _mm512_mullo_epi32(z13, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z13 = _mm512_sub_epi32(z5, z16); + z5 = _mm512_add_epi32(z5, z16); + z16 = _mm512_mullo_epi32(z14, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z14 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z16 = _mm512_mullo_epi32(z15, z21); + z17 = _mm512_add_epi32(z16, z22); + z17 = _mm512_srai_epi32(z17, 23); + z17 = _mm512_mullo_epi32(z17, z23); + z16 = _mm512_sub_epi32(z16, z17); + z15 = _mm512_sub_epi32(z7, z16); + z7 = _mm512_add_epi32(z7, z16); + /* len = 64 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z16 = _mm512_mullo_epi32(z4, z22); + z17 = _mm512_shuffle_epi32(z4, 0xf5); + z4 = _mm512_mul_epi32(z4, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z4, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z4 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z16 = _mm512_mullo_epi32(z6, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z6 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z3, z16); + z3 = _mm512_add_epi32(z3, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z16 = _mm512_mullo_epi32(z12, z22); + z17 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z12, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z12 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z9, z16); + z9 = _mm512_add_epi32(z9, z16); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z11, z16); + z11 = _mm512_add_epi32(z11, z16); + /* len = 32 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z16 = _mm512_mullo_epi32(z2, z22); + z17 = _mm512_shuffle_epi32(z2, 0xf5); + z2 = _mm512_mul_epi32(z2, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z2, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z2 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z1, z16); + z1 = _mm512_add_epi32(z1, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z16 = _mm512_mullo_epi32(z6, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z6 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z5, z16); + z5 = _mm512_add_epi32(z5, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z16 = _mm512_mullo_epi32(z10, z22); + z17 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z10, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z10 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z9, z16); + z9 = _mm512_add_epi32(z9, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z16 = _mm512_mullo_epi32(z14, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z14 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z13, z16); + z13 = _mm512_add_epi32(z13, z16); + /* len = 16 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1024)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1088)); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1152)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1216)); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1280)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1344)); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1408)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1472)); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1536)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1600)); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1664)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1728)); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1792)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1856)); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 8 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1920)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1984)); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2048)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2112)); + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2176)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2240)); + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2304)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2368)); + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2432)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2496)); + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2560)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2624)); + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2688)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2752)); + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2816)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2880)); + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 4 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2944)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3008)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z0, z24, z1); + z1 = z24; + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3072)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3136)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z2, z24, z3); + z3 = z24; + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3200)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3264)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z4, z24, z5); + z5 = z24; + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3328)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3392)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z6, z24, z7); + z7 = z24; + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3456)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3520)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z8, z16, z9); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z9 = z24; + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3584)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3648)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z10, z16, z11); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z10, z24, z11); + z11 = z24; + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3712)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3776)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z12, z16, z13); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z12, z24, z13); + z13 = z24; + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3840)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3904)); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z14, z16, z15); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z14, z24, z15); + z15 = z24; + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 2 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3968)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4032)); + z16 = _mm512_unpacklo_epi64(z0, z1); + z1 = _mm512_unpackhi_epi64(z0, z1); + z0 = z16; + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4096)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4160)); + z16 = _mm512_unpacklo_epi64(z2, z3); + z3 = _mm512_unpackhi_epi64(z2, z3); + z2 = z16; + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4224)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4288)); + z16 = _mm512_unpacklo_epi64(z4, z5); + z5 = _mm512_unpackhi_epi64(z4, z5); + z4 = z16; + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4352)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4416)); + z16 = _mm512_unpacklo_epi64(z6, z7); + z7 = _mm512_unpackhi_epi64(z6, z7); + z6 = z16; + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4480)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4544)); + z16 = _mm512_unpacklo_epi64(z8, z9); + z9 = _mm512_unpackhi_epi64(z8, z9); + z8 = z16; + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4608)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4672)); + z16 = _mm512_unpacklo_epi64(z10, z11); + z11 = _mm512_unpackhi_epi64(z10, z11); + z10 = z16; + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4736)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4800)); + z16 = _mm512_unpacklo_epi64(z12, z13); + z13 = _mm512_unpackhi_epi64(z12, z13); + z12 = z16; + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4864)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4928)); + z16 = _mm512_unpacklo_epi64(z14, z15); + z15 = _mm512_unpackhi_epi64(z14, z15); + z14 = z16; + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + /* len = 1 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4992)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5056)); + z16 = _mm512_slli_epi64(z1, 32); + z17 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z16); + z1 = _mm512_mask_blend_epi32(k2, z1, z17); + z16 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z1, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_sub_epi32(z0, z16); + z0 = _mm512_add_epi32(z0, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5120)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5184)); + z16 = _mm512_slli_epi64(z3, 32); + z17 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z16); + z3 = _mm512_mask_blend_epi32(k2, z3, z17); + z16 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z3, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_sub_epi32(z2, z16); + z2 = _mm512_add_epi32(z2, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5248)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5312)); + z16 = _mm512_slli_epi64(z5, 32); + z17 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z16); + z5 = _mm512_mask_blend_epi32(k2, z5, z17); + z16 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z5, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_sub_epi32(z4, z16); + z4 = _mm512_add_epi32(z4, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5376)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5440)); + z16 = _mm512_slli_epi64(z7, 32); + z17 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z16); + z7 = _mm512_mask_blend_epi32(k2, z7, z17); + z16 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z7, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_sub_epi32(z6, z16); + z6 = _mm512_add_epi32(z6, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5504)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5568)); + z16 = _mm512_slli_epi64(z9, 32); + z17 = _mm512_srli_epi64(z8, 32); + z8 = _mm512_mask_blend_epi32(k1, z8, z16); + z9 = _mm512_mask_blend_epi32(k2, z9, z17); + z16 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z9, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_sub_epi32(z8, z16); + z8 = _mm512_add_epi32(z8, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5632)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5696)); + z16 = _mm512_slli_epi64(z11, 32); + z17 = _mm512_srli_epi64(z10, 32); + z10 = _mm512_mask_blend_epi32(k1, z10, z16); + z11 = _mm512_mask_blend_epi32(k2, z11, z17); + z16 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z11, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_sub_epi32(z10, z16); + z10 = _mm512_add_epi32(z10, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5760)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5824)); + z16 = _mm512_slli_epi64(z13, 32); + z17 = _mm512_srli_epi64(z12, 32); + z12 = _mm512_mask_blend_epi32(k1, z12, z16); + z13 = _mm512_mask_blend_epi32(k2, z13, z17); + z16 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z13, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_sub_epi32(z12, z16); + z12 = _mm512_add_epi32(z12, z16); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5888)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5952)); + z16 = _mm512_slli_epi64(z15, 32); + z17 = _mm512_srli_epi64(z14, 32); + z14 = _mm512_mask_blend_epi32(k1, z14, z16); + z15 = _mm512_mask_blend_epi32(k2, z15, z17); + z16 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z15, z16); + z18 = _mm512_sub_epi64(z17, z18); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_sub_epi32(z14, z16); + z14 = _mm512_add_epi32(z14, z16); + z16 = _mm512_unpacklo_epi32(z0, z1); + z17 = _mm512_unpackhi_epi32(z0, z1); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z1 = z28; + z1 = _mm512_permutex2var_epi64(z16, z1, z17); + z0 = z18; + z16 = _mm512_unpacklo_epi32(z2, z3); + z17 = _mm512_unpackhi_epi32(z2, z3); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z3 = z28; + z3 = _mm512_permutex2var_epi64(z16, z3, z17); + z2 = z18; + z16 = _mm512_unpacklo_epi32(z4, z5); + z17 = _mm512_unpackhi_epi32(z4, z5); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z5 = z28; + z5 = _mm512_permutex2var_epi64(z16, z5, z17); + z4 = z18; + z16 = _mm512_unpacklo_epi32(z6, z7); + z17 = _mm512_unpackhi_epi32(z6, z7); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z7 = z28; + z7 = _mm512_permutex2var_epi64(z16, z7, z17); + z6 = z18; + z16 = _mm512_unpacklo_epi32(z8, z9); + z17 = _mm512_unpackhi_epi32(z8, z9); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z9 = z28; + z9 = _mm512_permutex2var_epi64(z16, z9, z17); + z8 = z18; + z16 = _mm512_unpacklo_epi32(z10, z11); + z17 = _mm512_unpackhi_epi32(z10, z11); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z11 = z28; + z11 = _mm512_permutex2var_epi64(z16, z11, z17); + z10 = z18; + z16 = _mm512_unpacklo_epi32(z12, z13); + z17 = _mm512_unpackhi_epi32(z12, z13); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z13 = z28; + z13 = _mm512_permutex2var_epi64(z16, z13, z17); + z12 = z18; + z16 = _mm512_unpacklo_epi32(z14, z15); + z17 = _mm512_unpackhi_epi32(z14, z15); + z18 = z27; + z18 = _mm512_permutex2var_epi64(z16, z18, z17); + z15 = z28; + z15 = _mm512_permutex2var_epi64(z16, z15, z17); + z14 = z18; + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_invntt_1p_avx512(sword32* r) +{ + word64 rdi, rax, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23, z24, z25, z26; + __mmask16 k1, k2; + + rdi = (word64)(size_t)r; + + rax = (word64)(0x7fe001); + z23 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z23 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z23)); + rax = (word32)(0xaaaa); + k1 = (__mmask16)(word32)rax; + rax = (word32)(0x5555); + k2 = (__mmask16)(word32)rax; + z25 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm20, 0)); + z26 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm31, 0)); + /* invntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx512_zetas_inv_1p); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + /* len = 1 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z16 = _mm512_slli_epi64(z1, 32); + z17 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z16); + z1 = _mm512_mask_blend_epi32(k2, z1, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z16 = _mm512_slli_epi64(z3, 32); + z17 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z16); + z3 = _mm512_mask_blend_epi32(k2, z3, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = _mm512_slli_epi64(z5, 32); + z17 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z16); + z5 = _mm512_mask_blend_epi32(k2, z5, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z16 = _mm512_slli_epi64(z7, 32); + z17 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z16); + z7 = _mm512_mask_blend_epi32(k2, z7, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = _mm512_slli_epi64(z9, 32); + z17 = _mm512_srli_epi64(z8, 32); + z8 = _mm512_mask_blend_epi32(k1, z8, z16); + z9 = _mm512_mask_blend_epi32(k2, z9, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = _mm512_slli_epi64(z11, 32); + z17 = _mm512_srli_epi64(z10, 32); + z10 = _mm512_mask_blend_epi32(k1, z10, z16); + z11 = _mm512_mask_blend_epi32(k2, z11, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_slli_epi64(z13, 32); + z17 = _mm512_srli_epi64(z12, 32); + z12 = _mm512_mask_blend_epi32(k1, z12, z16); + z13 = _mm512_mask_blend_epi32(k2, z13, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z16 = _mm512_slli_epi64(z15, 32); + z17 = _mm512_srli_epi64(z14, 32); + z14 = _mm512_mask_blend_epi32(k1, z14, z16); + z15 = _mm512_mask_blend_epi32(k2, z15, z17); + /* len = 2 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1024)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1088)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z16 = _mm512_unpacklo_epi64(z0, z1); + z1 = _mm512_unpackhi_epi64(z0, z1); + z0 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1152)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1216)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z16 = _mm512_unpacklo_epi64(z2, z3); + z3 = _mm512_unpackhi_epi64(z2, z3); + z2 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1280)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1344)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = _mm512_unpacklo_epi64(z4, z5); + z5 = _mm512_unpackhi_epi64(z4, z5); + z4 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1408)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1472)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z16 = _mm512_unpacklo_epi64(z6, z7); + z7 = _mm512_unpackhi_epi64(z6, z7); + z6 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1536)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1600)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = _mm512_unpacklo_epi64(z8, z9); + z9 = _mm512_unpackhi_epi64(z8, z9); + z8 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1664)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1728)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = _mm512_unpacklo_epi64(z10, z11); + z11 = _mm512_unpackhi_epi64(z10, z11); + z10 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1792)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1856)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_unpacklo_epi64(z12, z13); + z13 = _mm512_unpackhi_epi64(z12, z13); + z12 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1920)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1984)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z16 = _mm512_unpacklo_epi64(z14, z15); + z15 = _mm512_unpackhi_epi64(z14, z15); + z14 = z16; + /* len = 4 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2048)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2112)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z0, z24, z1); + z1 = z24; + z0 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2176)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2240)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z2, z24, z3); + z3 = z24; + z2 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2304)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2368)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z4, z24, z5); + z5 = z24; + z4 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2432)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2496)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z6, z24, z7); + z7 = z24; + z6 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2560)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2624)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z8, z16, z9); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z9 = z24; + z8 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2688)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2752)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z10, z16, z11); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z10, z24, z11); + z11 = z24; + z10 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2816)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2880)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z12, z16, z13); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z12, z24, z13); + z13 = z24; + z12 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2944)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3008)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z14, z16, z15); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z14, z24, z15); + z15 = z24; + z14 = z16; + /* len = 8 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3072)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3136)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3200)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3264)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3328)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3392)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3456)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3520)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3584)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3648)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3712)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3776)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3840)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3904)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3968)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4032)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + /* len = 16 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4096)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4160)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4224)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4288)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4352)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4416)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4480)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4544)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4608)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4672)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4736)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4800)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4864)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4928)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4992)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5056)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + /* len = 32 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5120)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5184)); + z16 = _mm512_sub_epi32(z0, z2); + z0 = _mm512_add_epi32(z0, z2); + z2 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z2, 0xf5); + z2 = _mm512_mul_epi32(z2, z23); + z18 = _mm512_mul_epi32(z18, z23); + z2 = _mm512_sub_epi64(z16, z2); + z18 = _mm512_sub_epi64(z17, z18); + z2 = _mm512_shuffle_epi32(z2, 0xf5); + z2 = _mm512_mask_blend_epi32(k1, z2, z18); + z16 = _mm512_sub_epi32(z1, z3); + z1 = _mm512_add_epi32(z1, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5248)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5312)); + z16 = _mm512_sub_epi32(z4, z6); + z4 = _mm512_add_epi32(z4, z6); + z6 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z23); + z18 = _mm512_mul_epi32(z18, z23); + z6 = _mm512_sub_epi64(z16, z6); + z18 = _mm512_sub_epi64(z17, z18); + z6 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mask_blend_epi32(k1, z6, z18); + z16 = _mm512_sub_epi32(z5, z7); + z5 = _mm512_add_epi32(z5, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5376)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5440)); + z16 = _mm512_sub_epi32(z8, z10); + z8 = _mm512_add_epi32(z8, z10); + z10 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z23); + z18 = _mm512_mul_epi32(z18, z23); + z10 = _mm512_sub_epi64(z16, z10); + z18 = _mm512_sub_epi64(z17, z18); + z10 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mask_blend_epi32(k1, z10, z18); + z16 = _mm512_sub_epi32(z9, z11); + z9 = _mm512_add_epi32(z9, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5504)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5568)); + z16 = _mm512_sub_epi32(z12, z14); + z12 = _mm512_add_epi32(z12, z14); + z14 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z23); + z18 = _mm512_mul_epi32(z18, z23); + z14 = _mm512_sub_epi64(z16, z14); + z18 = _mm512_sub_epi64(z17, z18); + z14 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mask_blend_epi32(k1, z14, z18); + z16 = _mm512_sub_epi32(z13, z15); + z13 = _mm512_add_epi32(z13, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + /* len = 64 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5632)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5696)); + z16 = _mm512_sub_epi32(z0, z4); + z0 = _mm512_add_epi32(z0, z4); + z4 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z4, 0xf5); + z4 = _mm512_mul_epi32(z4, z23); + z18 = _mm512_mul_epi32(z18, z23); + z4 = _mm512_sub_epi64(z16, z4); + z18 = _mm512_sub_epi64(z17, z18); + z4 = _mm512_shuffle_epi32(z4, 0xf5); + z4 = _mm512_mask_blend_epi32(k1, z4, z18); + z16 = _mm512_sub_epi32(z1, z5); + z1 = _mm512_add_epi32(z1, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = _mm512_sub_epi32(z2, z6); + z2 = _mm512_add_epi32(z2, z6); + z6 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z23); + z18 = _mm512_mul_epi32(z18, z23); + z6 = _mm512_sub_epi64(z16, z6); + z18 = _mm512_sub_epi64(z17, z18); + z6 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mask_blend_epi32(k1, z6, z18); + z16 = _mm512_sub_epi32(z3, z7); + z3 = _mm512_add_epi32(z3, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5760)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5824)); + z16 = _mm512_sub_epi32(z8, z12); + z8 = _mm512_add_epi32(z8, z12); + z12 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z23); + z18 = _mm512_mul_epi32(z18, z23); + z12 = _mm512_sub_epi64(z16, z12); + z18 = _mm512_sub_epi64(z17, z18); + z12 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mask_blend_epi32(k1, z12, z18); + z16 = _mm512_sub_epi32(z9, z13); + z9 = _mm512_add_epi32(z9, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_sub_epi32(z10, z14); + z10 = _mm512_add_epi32(z10, z14); + z14 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z23); + z18 = _mm512_mul_epi32(z18, z23); + z14 = _mm512_sub_epi64(z16, z14); + z18 = _mm512_sub_epi64(z17, z18); + z14 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mask_blend_epi32(k1, z14, z18); + z16 = _mm512_sub_epi32(z11, z15); + z11 = _mm512_add_epi32(z11, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + /* len = 128 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5888)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5952)); + z16 = _mm512_sub_epi32(z0, z8); + z0 = _mm512_add_epi32(z0, z8); + z8 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z8, 0xf5); + z8 = _mm512_mul_epi32(z8, z23); + z18 = _mm512_mul_epi32(z18, z23); + z8 = _mm512_sub_epi64(z16, z8); + z18 = _mm512_sub_epi64(z17, z18); + z8 = _mm512_shuffle_epi32(z8, 0xf5); + z8 = _mm512_mask_blend_epi32(k1, z8, z18); + z16 = _mm512_sub_epi32(z1, z9); + z1 = _mm512_add_epi32(z1, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = _mm512_sub_epi32(z2, z10); + z2 = _mm512_add_epi32(z2, z10); + z10 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z23); + z18 = _mm512_mul_epi32(z18, z23); + z10 = _mm512_sub_epi64(z16, z10); + z18 = _mm512_sub_epi64(z17, z18); + z10 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mask_blend_epi32(k1, z10, z18); + z16 = _mm512_sub_epi32(z3, z11); + z3 = _mm512_add_epi32(z3, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = _mm512_sub_epi32(z4, z12); + z4 = _mm512_add_epi32(z4, z12); + z12 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z23); + z18 = _mm512_mul_epi32(z18, z23); + z12 = _mm512_sub_epi64(z16, z12); + z18 = _mm512_sub_epi64(z17, z18); + z12 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mask_blend_epi32(k1, z12, z18); + z16 = _mm512_sub_epi32(z5, z13); + z5 = _mm512_add_epi32(z5, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_sub_epi32(z6, z14); + z6 = _mm512_add_epi32(z6, z14); + z14 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z23); + z18 = _mm512_mul_epi32(z18, z23); + z14 = _mm512_sub_epi64(z16, z14); + z18 = _mm512_sub_epi64(z17, z18); + z14 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mask_blend_epi32(k1, z14, z18); + z16 = _mm512_sub_epi32(z7, z15); + z7 = _mm512_add_epi32(z7, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6016)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6080)); + z16 = _mm512_mullo_epi32(z0, z22); + z19 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z0, 0xf5); + z20 = _mm512_shuffle_epi32(z1, 0xf5); + z0 = _mm512_mul_epi32(z0, z21); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z0, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z1, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z0 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z2, z22); + z19 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z2, 0xf5); + z20 = _mm512_shuffle_epi32(z3, 0xf5); + z2 = _mm512_mul_epi32(z2, z21); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z2, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z3, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z2 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z4, z22); + z19 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z4, 0xf5); + z20 = _mm512_shuffle_epi32(z5, 0xf5); + z4 = _mm512_mul_epi32(z4, z21); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z4, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z5, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z4 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z6, z22); + z19 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z20 = _mm512_shuffle_epi32(z7, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z7, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z6 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z8, z22); + z19 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z8, 0xf5); + z20 = _mm512_shuffle_epi32(z9, 0xf5); + z8 = _mm512_mul_epi32(z8, z21); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z8, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z9, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z8 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z10, z22); + z19 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z10, 0xf5); + z20 = _mm512_shuffle_epi32(z11, 0xf5); + z10 = _mm512_mul_epi32(z10, z21); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z10, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z11, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z10 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z12, z22); + z19 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z12, 0xf5); + z20 = _mm512_shuffle_epi32(z13, 0xf5); + z12 = _mm512_mul_epi32(z12, z21); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z12, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z13, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z12 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z14, z22); + z19 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z20 = _mm512_shuffle_epi32(z15, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z15, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z14 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_mask_blend_epi32(k1, z19, z17); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_invntt_full_1p_avx512(sword32* r) +{ + word64 rdi, rax, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23, z24, z25, z26; + __mmask16 k1, k2; + + rdi = (word64)(size_t)r; + + rax = (word64)(0x7fe001); + z23 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z23 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z23)); + rax = (word32)(0xaaaa); + k1 = (__mmask16)(word32)rax; + rax = (word32)(0x5555); + k2 = (__mmask16)(word32)rax; + z25 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm20, 0)); + z26 = _mm512_loadu_si512((const void*)WC_PR(L_mldsa_avx512_perm31, 0)); + /* invntt */ + rdx = (word64)((word64)(size_t)L_mldsa_avx512_zetas_inv_1p); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z16 = z25; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z0, z24, z1); + z1 = z24; + z0 = z16; + z16 = _mm512_unpacklo_epi64(z0, z1); + z1 = _mm512_unpackhi_epi64(z0, z1); + z0 = z16; + z16 = _mm512_slli_epi64(z1, 32); + z17 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z16); + z1 = _mm512_mask_blend_epi32(k2, z1, z17); + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z16 = z25; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z2, z24, z3); + z3 = z24; + z2 = z16; + z16 = _mm512_unpacklo_epi64(z2, z3); + z3 = _mm512_unpackhi_epi64(z2, z3); + z2 = z16; + z16 = _mm512_slli_epi64(z3, 32); + z17 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z16); + z3 = _mm512_mask_blend_epi32(k2, z3, z17); + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z16 = z25; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z4, z24, z5); + z5 = z24; + z4 = z16; + z16 = _mm512_unpacklo_epi64(z4, z5); + z5 = _mm512_unpackhi_epi64(z4, z5); + z4 = z16; + z16 = _mm512_slli_epi64(z5, 32); + z17 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z16); + z5 = _mm512_mask_blend_epi32(k2, z5, z17); + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z16 = z25; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z6, z24, z7); + z7 = z24; + z6 = z16; + z16 = _mm512_unpacklo_epi64(z6, z7); + z7 = _mm512_unpackhi_epi64(z6, z7); + z6 = z16; + z16 = _mm512_slli_epi64(z7, 32); + z17 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z16); + z7 = _mm512_mask_blend_epi32(k2, z7, z17); + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z16 = z25; + z16 = _mm512_permutex2var_epi64(z8, z16, z9); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z9 = z24; + z8 = z16; + z16 = _mm512_unpacklo_epi64(z8, z9); + z9 = _mm512_unpackhi_epi64(z8, z9); + z8 = z16; + z16 = _mm512_slli_epi64(z9, 32); + z17 = _mm512_srli_epi64(z8, 32); + z8 = _mm512_mask_blend_epi32(k1, z8, z16); + z9 = _mm512_mask_blend_epi32(k2, z9, z17); + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z16 = z25; + z16 = _mm512_permutex2var_epi64(z10, z16, z11); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z10, z24, z11); + z11 = z24; + z10 = z16; + z16 = _mm512_unpacklo_epi64(z10, z11); + z11 = _mm512_unpackhi_epi64(z10, z11); + z10 = z16; + z16 = _mm512_slli_epi64(z11, 32); + z17 = _mm512_srli_epi64(z10, 32); + z10 = _mm512_mask_blend_epi32(k1, z10, z16); + z11 = _mm512_mask_blend_epi32(k2, z11, z17); + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z16 = z25; + z16 = _mm512_permutex2var_epi64(z12, z16, z13); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z12, z24, z13); + z13 = z24; + z12 = z16; + z16 = _mm512_unpacklo_epi64(z12, z13); + z13 = _mm512_unpackhi_epi64(z12, z13); + z12 = z16; + z16 = _mm512_slli_epi64(z13, 32); + z17 = _mm512_srli_epi64(z12, 32); + z12 = _mm512_mask_blend_epi32(k1, z12, z16); + z13 = _mm512_mask_blend_epi32(k2, z13, z17); + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + z16 = z25; + z16 = _mm512_permutex2var_epi64(z14, z16, z15); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z14, z24, z15); + z15 = z24; + z14 = z16; + z16 = _mm512_unpacklo_epi64(z14, z15); + z15 = _mm512_unpackhi_epi64(z14, z15); + z14 = z16; + z16 = _mm512_slli_epi64(z15, 32); + z17 = _mm512_srli_epi64(z14, 32); + z14 = _mm512_mask_blend_epi32(k1, z14, z16); + z15 = _mm512_mask_blend_epi32(k2, z15, z17); + /* len = 1 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z16 = _mm512_slli_epi64(z1, 32); + z17 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z16); + z1 = _mm512_mask_blend_epi32(k2, z1, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z16 = _mm512_slli_epi64(z3, 32); + z17 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z16); + z3 = _mm512_mask_blend_epi32(k2, z3, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = _mm512_slli_epi64(z5, 32); + z17 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z16); + z5 = _mm512_mask_blend_epi32(k2, z5, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z16 = _mm512_slli_epi64(z7, 32); + z17 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z16); + z7 = _mm512_mask_blend_epi32(k2, z7, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = _mm512_slli_epi64(z9, 32); + z17 = _mm512_srli_epi64(z8, 32); + z8 = _mm512_mask_blend_epi32(k1, z8, z16); + z9 = _mm512_mask_blend_epi32(k2, z9, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = _mm512_slli_epi64(z11, 32); + z17 = _mm512_srli_epi64(z10, 32); + z10 = _mm512_mask_blend_epi32(k1, z10, z16); + z11 = _mm512_mask_blend_epi32(k2, z11, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_slli_epi64(z13, 32); + z17 = _mm512_srli_epi64(z12, 32); + z12 = _mm512_mask_blend_epi32(k1, z12, z16); + z13 = _mm512_mask_blend_epi32(k2, z13, z17); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z21 = _mm512_shuffle_epi32(z21, 0xf5); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z16 = _mm512_slli_epi64(z15, 32); + z17 = _mm512_srli_epi64(z14, 32); + z14 = _mm512_mask_blend_epi32(k1, z14, z16); + z15 = _mm512_mask_blend_epi32(k2, z15, z17); + /* len = 2 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1024)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1088)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z16 = _mm512_unpacklo_epi64(z0, z1); + z1 = _mm512_unpackhi_epi64(z0, z1); + z0 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1152)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1216)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z16 = _mm512_unpacklo_epi64(z2, z3); + z3 = _mm512_unpackhi_epi64(z2, z3); + z2 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1280)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1344)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = _mm512_unpacklo_epi64(z4, z5); + z5 = _mm512_unpackhi_epi64(z4, z5); + z4 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1408)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1472)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z16 = _mm512_unpacklo_epi64(z6, z7); + z7 = _mm512_unpackhi_epi64(z6, z7); + z6 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1536)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1600)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = _mm512_unpacklo_epi64(z8, z9); + z9 = _mm512_unpackhi_epi64(z8, z9); + z8 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1664)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1728)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = _mm512_unpacklo_epi64(z10, z11); + z11 = _mm512_unpackhi_epi64(z10, z11); + z10 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1792)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1856)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_unpacklo_epi64(z12, z13); + z13 = _mm512_unpackhi_epi64(z12, z13); + z12 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1920)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1984)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z16 = _mm512_unpacklo_epi64(z14, z15); + z15 = _mm512_unpackhi_epi64(z14, z15); + z14 = z16; + /* len = 4 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2048)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2112)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z0, z24, z1); + z1 = z24; + z0 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2176)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2240)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z2, z24, z3); + z3 = z24; + z2 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2304)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2368)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z4, z24, z5); + z5 = z24; + z4 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2432)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2496)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z6, z24, z7); + z7 = z24; + z6 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2560)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2624)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z8, z16, z9); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z9 = z24; + z8 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2688)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2752)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z10, z16, z11); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z10, z24, z11); + z11 = z24; + z10 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2816)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2880)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z12, z16, z13); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z12, z24, z13); + z13 = z24; + z12 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2944)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3008)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z16 = z25; + z16 = _mm512_permutex2var_epi64(z14, z16, z15); + z24 = z26; + z24 = _mm512_permutex2var_epi64(z14, z24, z15); + z15 = z24; + z14 = z16; + /* len = 8 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3072)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3136)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z16 = _mm512_shuffle_i64x2(z0, z1, 0x44); + z1 = _mm512_shuffle_i64x2(z0, z1, 0xee); + z0 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3200)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3264)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z16 = _mm512_shuffle_i64x2(z2, z3, 0x44); + z3 = _mm512_shuffle_i64x2(z2, z3, 0xee); + z2 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3328)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3392)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = _mm512_shuffle_i64x2(z4, z5, 0x44); + z5 = _mm512_shuffle_i64x2(z4, z5, 0xee); + z4 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3456)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3520)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z16 = _mm512_shuffle_i64x2(z6, z7, 0x44); + z7 = _mm512_shuffle_i64x2(z6, z7, 0xee); + z6 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3584)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3648)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = _mm512_shuffle_i64x2(z8, z9, 0x44); + z9 = _mm512_shuffle_i64x2(z8, z9, 0xee); + z8 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3712)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3776)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = _mm512_shuffle_i64x2(z10, z11, 0x44); + z11 = _mm512_shuffle_i64x2(z10, z11, 0xee); + z10 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3840)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3904)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_shuffle_i64x2(z12, z13, 0x44); + z13 = _mm512_shuffle_i64x2(z12, z13, 0xee); + z12 = z16; + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3968)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4032)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z16 = _mm512_shuffle_i64x2(z14, z15, 0x44); + z15 = _mm512_shuffle_i64x2(z14, z15, 0xee); + z14 = z16; + /* len = 16 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4096)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4160)); + z16 = _mm512_sub_epi32(z0, z1); + z0 = _mm512_add_epi32(z0, z1); + z1 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mul_epi32(z1, z23); + z18 = _mm512_mul_epi32(z18, z23); + z1 = _mm512_sub_epi64(z16, z1); + z18 = _mm512_sub_epi64(z17, z18); + z1 = _mm512_shuffle_epi32(z1, 0xf5); + z1 = _mm512_mask_blend_epi32(k1, z1, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4224)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4288)); + z16 = _mm512_sub_epi32(z2, z3); + z2 = _mm512_add_epi32(z2, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4352)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4416)); + z16 = _mm512_sub_epi32(z4, z5); + z4 = _mm512_add_epi32(z4, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4480)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4544)); + z16 = _mm512_sub_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4608)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4672)); + z16 = _mm512_sub_epi32(z8, z9); + z8 = _mm512_add_epi32(z8, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4736)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4800)); + z16 = _mm512_sub_epi32(z10, z11); + z10 = _mm512_add_epi32(z10, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4864)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4928)); + z16 = _mm512_sub_epi32(z12, z13); + z12 = _mm512_add_epi32(z12, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4992)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5056)); + z16 = _mm512_sub_epi32(z14, z15); + z14 = _mm512_add_epi32(z14, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + /* len = 32 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5120)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5184)); + z16 = _mm512_sub_epi32(z0, z2); + z0 = _mm512_add_epi32(z0, z2); + z2 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z2, 0xf5); + z2 = _mm512_mul_epi32(z2, z23); + z18 = _mm512_mul_epi32(z18, z23); + z2 = _mm512_sub_epi64(z16, z2); + z18 = _mm512_sub_epi64(z17, z18); + z2 = _mm512_shuffle_epi32(z2, 0xf5); + z2 = _mm512_mask_blend_epi32(k1, z2, z18); + z16 = _mm512_sub_epi32(z1, z3); + z1 = _mm512_add_epi32(z1, z3); + z3 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mul_epi32(z3, z23); + z18 = _mm512_mul_epi32(z18, z23); + z3 = _mm512_sub_epi64(z16, z3); + z18 = _mm512_sub_epi64(z17, z18); + z3 = _mm512_shuffle_epi32(z3, 0xf5); + z3 = _mm512_mask_blend_epi32(k1, z3, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5248)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5312)); + z16 = _mm512_sub_epi32(z4, z6); + z4 = _mm512_add_epi32(z4, z6); + z6 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z23); + z18 = _mm512_mul_epi32(z18, z23); + z6 = _mm512_sub_epi64(z16, z6); + z18 = _mm512_sub_epi64(z17, z18); + z6 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mask_blend_epi32(k1, z6, z18); + z16 = _mm512_sub_epi32(z5, z7); + z5 = _mm512_add_epi32(z5, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5376)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5440)); + z16 = _mm512_sub_epi32(z8, z10); + z8 = _mm512_add_epi32(z8, z10); + z10 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z23); + z18 = _mm512_mul_epi32(z18, z23); + z10 = _mm512_sub_epi64(z16, z10); + z18 = _mm512_sub_epi64(z17, z18); + z10 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mask_blend_epi32(k1, z10, z18); + z16 = _mm512_sub_epi32(z9, z11); + z9 = _mm512_add_epi32(z9, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5504)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5568)); + z16 = _mm512_sub_epi32(z12, z14); + z12 = _mm512_add_epi32(z12, z14); + z14 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z23); + z18 = _mm512_mul_epi32(z18, z23); + z14 = _mm512_sub_epi64(z16, z14); + z18 = _mm512_sub_epi64(z17, z18); + z14 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mask_blend_epi32(k1, z14, z18); + z16 = _mm512_sub_epi32(z13, z15); + z13 = _mm512_add_epi32(z13, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + /* len = 64 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5632)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5696)); + z16 = _mm512_sub_epi32(z0, z4); + z0 = _mm512_add_epi32(z0, z4); + z4 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z4, 0xf5); + z4 = _mm512_mul_epi32(z4, z23); + z18 = _mm512_mul_epi32(z18, z23); + z4 = _mm512_sub_epi64(z16, z4); + z18 = _mm512_sub_epi64(z17, z18); + z4 = _mm512_shuffle_epi32(z4, 0xf5); + z4 = _mm512_mask_blend_epi32(k1, z4, z18); + z16 = _mm512_sub_epi32(z1, z5); + z1 = _mm512_add_epi32(z1, z5); + z5 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mul_epi32(z5, z23); + z18 = _mm512_mul_epi32(z18, z23); + z5 = _mm512_sub_epi64(z16, z5); + z18 = _mm512_sub_epi64(z17, z18); + z5 = _mm512_shuffle_epi32(z5, 0xf5); + z5 = _mm512_mask_blend_epi32(k1, z5, z18); + z16 = _mm512_sub_epi32(z2, z6); + z2 = _mm512_add_epi32(z2, z6); + z6 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mul_epi32(z6, z23); + z18 = _mm512_mul_epi32(z18, z23); + z6 = _mm512_sub_epi64(z16, z6); + z18 = _mm512_sub_epi64(z17, z18); + z6 = _mm512_shuffle_epi32(z6, 0xf5); + z6 = _mm512_mask_blend_epi32(k1, z6, z18); + z16 = _mm512_sub_epi32(z3, z7); + z3 = _mm512_add_epi32(z3, z7); + z7 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mul_epi32(z7, z23); + z18 = _mm512_mul_epi32(z18, z23); + z7 = _mm512_sub_epi64(z16, z7); + z18 = _mm512_sub_epi64(z17, z18); + z7 = _mm512_shuffle_epi32(z7, 0xf5); + z7 = _mm512_mask_blend_epi32(k1, z7, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5760)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5824)); + z16 = _mm512_sub_epi32(z8, z12); + z8 = _mm512_add_epi32(z8, z12); + z12 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z23); + z18 = _mm512_mul_epi32(z18, z23); + z12 = _mm512_sub_epi64(z16, z12); + z18 = _mm512_sub_epi64(z17, z18); + z12 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mask_blend_epi32(k1, z12, z18); + z16 = _mm512_sub_epi32(z9, z13); + z9 = _mm512_add_epi32(z9, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_sub_epi32(z10, z14); + z10 = _mm512_add_epi32(z10, z14); + z14 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z23); + z18 = _mm512_mul_epi32(z18, z23); + z14 = _mm512_sub_epi64(z16, z14); + z18 = _mm512_sub_epi64(z17, z18); + z14 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mask_blend_epi32(k1, z14, z18); + z16 = _mm512_sub_epi32(z11, z15); + z11 = _mm512_add_epi32(z11, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + /* len = 128 */ + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5888)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5952)); + z16 = _mm512_sub_epi32(z0, z8); + z0 = _mm512_add_epi32(z0, z8); + z8 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z8, 0xf5); + z8 = _mm512_mul_epi32(z8, z23); + z18 = _mm512_mul_epi32(z18, z23); + z8 = _mm512_sub_epi64(z16, z8); + z18 = _mm512_sub_epi64(z17, z18); + z8 = _mm512_shuffle_epi32(z8, 0xf5); + z8 = _mm512_mask_blend_epi32(k1, z8, z18); + z16 = _mm512_sub_epi32(z1, z9); + z1 = _mm512_add_epi32(z1, z9); + z9 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mul_epi32(z9, z23); + z18 = _mm512_mul_epi32(z18, z23); + z9 = _mm512_sub_epi64(z16, z9); + z18 = _mm512_sub_epi64(z17, z18); + z9 = _mm512_shuffle_epi32(z9, 0xf5); + z9 = _mm512_mask_blend_epi32(k1, z9, z18); + z16 = _mm512_sub_epi32(z2, z10); + z2 = _mm512_add_epi32(z2, z10); + z10 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mul_epi32(z10, z23); + z18 = _mm512_mul_epi32(z18, z23); + z10 = _mm512_sub_epi64(z16, z10); + z18 = _mm512_sub_epi64(z17, z18); + z10 = _mm512_shuffle_epi32(z10, 0xf5); + z10 = _mm512_mask_blend_epi32(k1, z10, z18); + z16 = _mm512_sub_epi32(z3, z11); + z3 = _mm512_add_epi32(z3, z11); + z11 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mul_epi32(z11, z23); + z18 = _mm512_mul_epi32(z18, z23); + z11 = _mm512_sub_epi64(z16, z11); + z18 = _mm512_sub_epi64(z17, z18); + z11 = _mm512_shuffle_epi32(z11, 0xf5); + z11 = _mm512_mask_blend_epi32(k1, z11, z18); + z16 = _mm512_sub_epi32(z4, z12); + z4 = _mm512_add_epi32(z4, z12); + z12 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mul_epi32(z12, z23); + z18 = _mm512_mul_epi32(z18, z23); + z12 = _mm512_sub_epi64(z16, z12); + z18 = _mm512_sub_epi64(z17, z18); + z12 = _mm512_shuffle_epi32(z12, 0xf5); + z12 = _mm512_mask_blend_epi32(k1, z12, z18); + z16 = _mm512_sub_epi32(z5, z13); + z5 = _mm512_add_epi32(z5, z13); + z13 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mul_epi32(z13, z23); + z18 = _mm512_mul_epi32(z18, z23); + z13 = _mm512_sub_epi64(z16, z13); + z18 = _mm512_sub_epi64(z17, z18); + z13 = _mm512_shuffle_epi32(z13, 0xf5); + z13 = _mm512_mask_blend_epi32(k1, z13, z18); + z16 = _mm512_sub_epi32(z6, z14); + z6 = _mm512_add_epi32(z6, z14); + z14 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mul_epi32(z14, z23); + z18 = _mm512_mul_epi32(z18, z23); + z14 = _mm512_sub_epi64(z16, z14); + z18 = _mm512_sub_epi64(z17, z18); + z14 = _mm512_shuffle_epi32(z14, 0xf5); + z14 = _mm512_mask_blend_epi32(k1, z14, z18); + z16 = _mm512_sub_epi32(z7, z15); + z7 = _mm512_add_epi32(z7, z15); + z15 = _mm512_mullo_epi32(z16, z22); + z17 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z21); + z17 = _mm512_mul_epi32(z17, z21); + z18 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mul_epi32(z15, z23); + z18 = _mm512_mul_epi32(z18, z23); + z15 = _mm512_sub_epi64(z16, z15); + z18 = _mm512_sub_epi64(z17, z18); + z15 = _mm512_shuffle_epi32(z15, 0xf5); + z15 = _mm512_mask_blend_epi32(k1, z15, z18); + z21 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6016)); + z22 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6080)); + z16 = _mm512_mullo_epi32(z0, z22); + z19 = _mm512_mullo_epi32(z1, z22); + z17 = _mm512_shuffle_epi32(z0, 0xf5); + z20 = _mm512_shuffle_epi32(z1, 0xf5); + z0 = _mm512_mul_epi32(z0, z21); + z1 = _mm512_mul_epi32(z1, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z0, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z1, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z0 = _mm512_mask_blend_epi32(k1, z16, z18); + z1 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z2, z22); + z19 = _mm512_mullo_epi32(z3, z22); + z17 = _mm512_shuffle_epi32(z2, 0xf5); + z20 = _mm512_shuffle_epi32(z3, 0xf5); + z2 = _mm512_mul_epi32(z2, z21); + z3 = _mm512_mul_epi32(z3, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z2, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z3, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z2 = _mm512_mask_blend_epi32(k1, z16, z18); + z3 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z4, z22); + z19 = _mm512_mullo_epi32(z5, z22); + z17 = _mm512_shuffle_epi32(z4, 0xf5); + z20 = _mm512_shuffle_epi32(z5, 0xf5); + z4 = _mm512_mul_epi32(z4, z21); + z5 = _mm512_mul_epi32(z5, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z4, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z5, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z4 = _mm512_mask_blend_epi32(k1, z16, z18); + z5 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z6, z22); + z19 = _mm512_mullo_epi32(z7, z22); + z17 = _mm512_shuffle_epi32(z6, 0xf5); + z20 = _mm512_shuffle_epi32(z7, 0xf5); + z6 = _mm512_mul_epi32(z6, z21); + z7 = _mm512_mul_epi32(z7, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z6, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z7, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z6 = _mm512_mask_blend_epi32(k1, z16, z18); + z7 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z8, z22); + z19 = _mm512_mullo_epi32(z9, z22); + z17 = _mm512_shuffle_epi32(z8, 0xf5); + z20 = _mm512_shuffle_epi32(z9, 0xf5); + z8 = _mm512_mul_epi32(z8, z21); + z9 = _mm512_mul_epi32(z9, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z8, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z9, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z8 = _mm512_mask_blend_epi32(k1, z16, z18); + z9 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z10, z22); + z19 = _mm512_mullo_epi32(z11, z22); + z17 = _mm512_shuffle_epi32(z10, 0xf5); + z20 = _mm512_shuffle_epi32(z11, 0xf5); + z10 = _mm512_mul_epi32(z10, z21); + z11 = _mm512_mul_epi32(z11, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z10, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z11, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z10 = _mm512_mask_blend_epi32(k1, z16, z18); + z11 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z12, z22); + z19 = _mm512_mullo_epi32(z13, z22); + z17 = _mm512_shuffle_epi32(z12, 0xf5); + z20 = _mm512_shuffle_epi32(z13, 0xf5); + z12 = _mm512_mul_epi32(z12, z21); + z13 = _mm512_mul_epi32(z13, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z12, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z13, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z12 = _mm512_mask_blend_epi32(k1, z16, z18); + z13 = _mm512_mask_blend_epi32(k1, z19, z17); + z16 = _mm512_mullo_epi32(z14, z22); + z19 = _mm512_mullo_epi32(z15, z22); + z17 = _mm512_shuffle_epi32(z14, 0xf5); + z20 = _mm512_shuffle_epi32(z15, 0xf5); + z14 = _mm512_mul_epi32(z14, z21); + z15 = _mm512_mul_epi32(z15, z21); + z17 = _mm512_mul_epi32(z17, z21); + z20 = _mm512_mul_epi32(z20, z21); + z18 = _mm512_shuffle_epi32(z16, 0xf5); + z16 = _mm512_mul_epi32(z16, z23); + z18 = _mm512_mul_epi32(z18, z23); + z16 = _mm512_sub_epi64(z14, z16); + z18 = _mm512_sub_epi64(z17, z18); + z17 = _mm512_shuffle_epi32(z19, 0xf5); + z19 = _mm512_mul_epi32(z19, z23); + z17 = _mm512_mul_epi32(z17, z23); + z19 = _mm512_sub_epi64(z15, z19); + z17 = _mm512_sub_epi64(z20, z17); + z16 = _mm512_shuffle_epi32(z16, 0xf5); + z19 = _mm512_shuffle_epi32(z19, 0xf5); + z14 = _mm512_mask_blend_epi32(k1, z16, z18); + z15 = _mm512_mask_blend_epi32(k1, z19, z17); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z12); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z13); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z14); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z15); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_poly_red_avx512(sword32* a) +{ + word64 rdi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11; + + rdi = (word64)(size_t)a; + + rdx = (word64)(0x7fe001); + z10 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z10 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z10)); + rdx = (word64)(0x400000); + z11 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z11 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z11)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_add_epi32(z0, z11); + z9 = _mm512_add_epi32(z1, z11); + z8 = _mm512_srai_epi32(z8, 23); + z9 = _mm512_srai_epi32(z9, 23); + z8 = _mm512_mullo_epi32(z8, z10); + z9 = _mm512_mullo_epi32(z9, z10); + z0 = _mm512_sub_epi32(z0, z8); + z1 = _mm512_sub_epi32(z1, z9); + z8 = _mm512_add_epi32(z2, z11); + z9 = _mm512_add_epi32(z3, z11); + z8 = _mm512_srai_epi32(z8, 23); + z9 = _mm512_srai_epi32(z9, 23); + z8 = _mm512_mullo_epi32(z8, z10); + z9 = _mm512_mullo_epi32(z9, z10); + z2 = _mm512_sub_epi32(z2, z8); + z3 = _mm512_sub_epi32(z3, z9); + z8 = _mm512_add_epi32(z4, z11); + z9 = _mm512_add_epi32(z5, z11); + z8 = _mm512_srai_epi32(z8, 23); + z9 = _mm512_srai_epi32(z9, 23); + z8 = _mm512_mullo_epi32(z8, z10); + z9 = _mm512_mullo_epi32(z9, z10); + z4 = _mm512_sub_epi32(z4, z8); + z5 = _mm512_sub_epi32(z5, z9); + z8 = _mm512_add_epi32(z6, z11); + z9 = _mm512_add_epi32(z7, z11); + z8 = _mm512_srai_epi32(z8, 23); + z9 = _mm512_srai_epi32(z9, 23); + z8 = _mm512_mullo_epi32(z8, z10); + z9 = _mm512_mullo_epi32(z9, z10); + z6 = _mm512_sub_epi32(z6, z8); + z7 = _mm512_sub_epi32(z7, z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_add_epi32(z0, z11); + z9 = _mm512_add_epi32(z1, z11); + z8 = _mm512_srai_epi32(z8, 23); + z9 = _mm512_srai_epi32(z9, 23); + z8 = _mm512_mullo_epi32(z8, z10); + z9 = _mm512_mullo_epi32(z9, z10); + z0 = _mm512_sub_epi32(z0, z8); + z1 = _mm512_sub_epi32(z1, z9); + z8 = _mm512_add_epi32(z2, z11); + z9 = _mm512_add_epi32(z3, z11); + z8 = _mm512_srai_epi32(z8, 23); + z9 = _mm512_srai_epi32(z9, 23); + z8 = _mm512_mullo_epi32(z8, z10); + z9 = _mm512_mullo_epi32(z9, z10); + z2 = _mm512_sub_epi32(z2, z8); + z3 = _mm512_sub_epi32(z3, z9); + z8 = _mm512_add_epi32(z4, z11); + z9 = _mm512_add_epi32(z5, z11); + z8 = _mm512_srai_epi32(z8, 23); + z9 = _mm512_srai_epi32(z9, 23); + z8 = _mm512_mullo_epi32(z8, z10); + z9 = _mm512_mullo_epi32(z9, z10); + z4 = _mm512_sub_epi32(z4, z8); + z5 = _mm512_sub_epi32(z5, z9); + z8 = _mm512_add_epi32(z6, z11); + z9 = _mm512_add_epi32(z7, z11); + z8 = _mm512_srai_epi32(z8, 23); + z9 = _mm512_srai_epi32(z9, 23); + z8 = _mm512_mullo_epi32(z8, z10); + z9 = _mm512_mullo_epi32(z9, z10); + z6 = _mm512_sub_epi32(z6, z8); + z7 = _mm512_sub_epi32(z7, z9); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z7); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_mul_avx512(sword32* r, const sword32* a, + const sword32* b) +{ + word64 rdi, rsi, rdx, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(0x7fe001); + z8 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z8 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z8)); + rax = (word64)(0x3802001); + z9 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z9 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z9)); + /* 0..31 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z0 = _mm512_mul_epi32(z0, z4); + z1 = _mm512_mul_epi32(z1, z5); + z2 = _mm512_mul_epi32(z2, z6); + z3 = _mm512_mul_epi32(z3, z7); + /* Mont Reduce */ + z4 = _mm512_mullo_epi32(z0, z9); + z5 = _mm512_mullo_epi32(z1, z9); + z6 = _mm512_mullo_epi32(z2, z9); + z7 = _mm512_mullo_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z8); + z5 = _mm512_mul_epi32(z5, z8); + z6 = _mm512_mul_epi32(z6, z8); + z7 = _mm512_mul_epi32(z7, z8); + z0 = _mm512_sub_epi32(z0, z4); + z1 = _mm512_sub_epi32(z1, z5); + z2 = _mm512_sub_epi32(z2, z6); + z3 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi64(z0, 32); + z2 = _mm512_srli_epi64(z2, 32); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_or_si512(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z2); + /* 32..63 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z0 = _mm512_mul_epi32(z0, z4); + z1 = _mm512_mul_epi32(z1, z5); + z2 = _mm512_mul_epi32(z2, z6); + z3 = _mm512_mul_epi32(z3, z7); + /* Mont Reduce */ + z4 = _mm512_mullo_epi32(z0, z9); + z5 = _mm512_mullo_epi32(z1, z9); + z6 = _mm512_mullo_epi32(z2, z9); + z7 = _mm512_mullo_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z8); + z5 = _mm512_mul_epi32(z5, z8); + z6 = _mm512_mul_epi32(z6, z8); + z7 = _mm512_mul_epi32(z7, z8); + z0 = _mm512_sub_epi32(z0, z4); + z1 = _mm512_sub_epi32(z1, z5); + z2 = _mm512_sub_epi32(z2, z6); + z3 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi64(z0, 32); + z2 = _mm512_srli_epi64(z2, 32); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_or_si512(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z2); + /* 64..95 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z0 = _mm512_mul_epi32(z0, z4); + z1 = _mm512_mul_epi32(z1, z5); + z2 = _mm512_mul_epi32(z2, z6); + z3 = _mm512_mul_epi32(z3, z7); + /* Mont Reduce */ + z4 = _mm512_mullo_epi32(z0, z9); + z5 = _mm512_mullo_epi32(z1, z9); + z6 = _mm512_mullo_epi32(z2, z9); + z7 = _mm512_mullo_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z8); + z5 = _mm512_mul_epi32(z5, z8); + z6 = _mm512_mul_epi32(z6, z8); + z7 = _mm512_mul_epi32(z7, z8); + z0 = _mm512_sub_epi32(z0, z4); + z1 = _mm512_sub_epi32(z1, z5); + z2 = _mm512_sub_epi32(z2, z6); + z3 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi64(z0, 32); + z2 = _mm512_srli_epi64(z2, 32); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_or_si512(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z2); + /* 96..127 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z0 = _mm512_mul_epi32(z0, z4); + z1 = _mm512_mul_epi32(z1, z5); + z2 = _mm512_mul_epi32(z2, z6); + z3 = _mm512_mul_epi32(z3, z7); + /* Mont Reduce */ + z4 = _mm512_mullo_epi32(z0, z9); + z5 = _mm512_mullo_epi32(z1, z9); + z6 = _mm512_mullo_epi32(z2, z9); + z7 = _mm512_mullo_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z8); + z5 = _mm512_mul_epi32(z5, z8); + z6 = _mm512_mul_epi32(z6, z8); + z7 = _mm512_mul_epi32(z7, z8); + z0 = _mm512_sub_epi32(z0, z4); + z1 = _mm512_sub_epi32(z1, z5); + z2 = _mm512_sub_epi32(z2, z6); + z3 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi64(z0, 32); + z2 = _mm512_srli_epi64(z2, 32); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_or_si512(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z2); + /* 128..159 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 512)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 576)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z0 = _mm512_mul_epi32(z0, z4); + z1 = _mm512_mul_epi32(z1, z5); + z2 = _mm512_mul_epi32(z2, z6); + z3 = _mm512_mul_epi32(z3, z7); + /* Mont Reduce */ + z4 = _mm512_mullo_epi32(z0, z9); + z5 = _mm512_mullo_epi32(z1, z9); + z6 = _mm512_mullo_epi32(z2, z9); + z7 = _mm512_mullo_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z8); + z5 = _mm512_mul_epi32(z5, z8); + z6 = _mm512_mul_epi32(z6, z8); + z7 = _mm512_mul_epi32(z7, z8); + z0 = _mm512_sub_epi32(z0, z4); + z1 = _mm512_sub_epi32(z1, z5); + z2 = _mm512_sub_epi32(z2, z6); + z3 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi64(z0, 32); + z2 = _mm512_srli_epi64(z2, 32); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_or_si512(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z2); + /* 160..191 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 640)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z0 = _mm512_mul_epi32(z0, z4); + z1 = _mm512_mul_epi32(z1, z5); + z2 = _mm512_mul_epi32(z2, z6); + z3 = _mm512_mul_epi32(z3, z7); + /* Mont Reduce */ + z4 = _mm512_mullo_epi32(z0, z9); + z5 = _mm512_mullo_epi32(z1, z9); + z6 = _mm512_mullo_epi32(z2, z9); + z7 = _mm512_mullo_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z8); + z5 = _mm512_mul_epi32(z5, z8); + z6 = _mm512_mul_epi32(z6, z8); + z7 = _mm512_mul_epi32(z7, z8); + z0 = _mm512_sub_epi32(z0, z4); + z1 = _mm512_sub_epi32(z1, z5); + z2 = _mm512_sub_epi32(z2, z6); + z3 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi64(z0, 32); + z2 = _mm512_srli_epi64(z2, 32); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_or_si512(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z2); + /* 192..223 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 768)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 832)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z0 = _mm512_mul_epi32(z0, z4); + z1 = _mm512_mul_epi32(z1, z5); + z2 = _mm512_mul_epi32(z2, z6); + z3 = _mm512_mul_epi32(z3, z7); + /* Mont Reduce */ + z4 = _mm512_mullo_epi32(z0, z9); + z5 = _mm512_mullo_epi32(z1, z9); + z6 = _mm512_mullo_epi32(z2, z9); + z7 = _mm512_mullo_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z8); + z5 = _mm512_mul_epi32(z5, z8); + z6 = _mm512_mul_epi32(z6, z8); + z7 = _mm512_mul_epi32(z7, z8); + z0 = _mm512_sub_epi32(z0, z4); + z1 = _mm512_sub_epi32(z1, z5); + z2 = _mm512_sub_epi32(z2, z6); + z3 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi64(z0, 32); + z2 = _mm512_srli_epi64(z2, 32); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_or_si512(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z2); + /* 224..255 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 896)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 960)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z0 = _mm512_mul_epi32(z0, z4); + z1 = _mm512_mul_epi32(z1, z5); + z2 = _mm512_mul_epi32(z2, z6); + z3 = _mm512_mul_epi32(z3, z7); + /* Mont Reduce */ + z4 = _mm512_mullo_epi32(z0, z9); + z5 = _mm512_mullo_epi32(z1, z9); + z6 = _mm512_mullo_epi32(z2, z9); + z7 = _mm512_mullo_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z8); + z5 = _mm512_mul_epi32(z5, z8); + z6 = _mm512_mul_epi32(z6, z8); + z7 = _mm512_mul_epi32(z7, z8); + z0 = _mm512_sub_epi32(z0, z4); + z1 = _mm512_sub_epi32(z1, z5); + z2 = _mm512_sub_epi32(z2, z6); + z3 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi64(z0, 32); + z2 = _mm512_srli_epi64(z2, 32); + z0 = _mm512_or_si512(z0, z1); + z2 = _mm512_or_si512(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z2); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_mul_vec_4_avx512(sword32* r, const sword32* a, + const sword32* b) +{ + word64 rdi, rsi, rdx, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(0x7fe001); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z12 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z12)); + rax = (word64)(0x3802001); + z13 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z13 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z13)); + /* 0..15 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1024)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1024)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2048)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3072)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2048)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3072)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + /* 16..31 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1088)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1088)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2112)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3136)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2112)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3136)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z0); + /* 32..47 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1152)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1152)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2176)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3200)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2176)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3200)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z0); + /* 48..63 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1216)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1216)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2240)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3264)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2240)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3264)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z0); + /* 64..79 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1280)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1280)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2304)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3328)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2304)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3328)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z0); + /* 80..95 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1344)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1344)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2368)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3392)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2368)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3392)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z0); + /* 96..111 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1408)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1408)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2432)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3456)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2432)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3456)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z0); + /* 112..127 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1472)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1472)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2496)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3520)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2496)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3520)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z0); + /* 128..143 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 512)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1536)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1536)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2560)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3584)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2560)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3584)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z0); + /* 144..159 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1600)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1600)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2624)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3648)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2624)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3648)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z0); + /* 160..175 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 640)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1664)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1664)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2688)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3712)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2688)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3712)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z0); + /* 176..191 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 704)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1728)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1728)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2752)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3776)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2752)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3776)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z0); + /* 192..207 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 768)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1792)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1792)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2816)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3840)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2816)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3840)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z0); + /* 208..223 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 832)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1856)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1856)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2880)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3904)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2880)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3904)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z0); + /* 224..239 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 896)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1920)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1920)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2944)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3968)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2944)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3968)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z0); + /* 240..255 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 960)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1984)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1984)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3008)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4032)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3008)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4032)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z0); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_mul_vec_5_avx512(sword32* r, const sword32* a, + const sword32* b) +{ + word64 rdi, rsi, rdx, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(0x7fe001); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z12 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z12)); + rax = (word64)(0x3802001); + z13 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z13 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z13)); + /* 0..15 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1024)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1024)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2048)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3072)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2048)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3072)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4096)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4096)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + /* 16..31 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1088)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1088)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2112)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3136)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2112)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3136)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4160)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4160)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z0); + /* 32..47 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1152)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1152)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2176)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3200)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2176)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3200)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4224)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4224)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z0); + /* 48..63 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1216)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1216)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2240)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3264)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2240)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3264)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4288)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4288)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z0); + /* 64..79 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1280)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1280)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2304)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3328)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2304)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3328)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4352)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4352)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z0); + /* 80..95 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1344)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1344)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2368)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3392)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2368)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3392)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4416)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4416)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z0); + /* 96..111 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1408)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1408)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2432)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3456)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2432)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3456)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4480)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4480)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z0); + /* 112..127 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1472)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1472)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2496)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3520)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2496)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3520)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4544)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4544)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z0); + /* 128..143 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 512)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1536)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1536)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2560)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3584)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2560)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3584)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4608)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4608)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z0); + /* 144..159 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1600)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1600)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2624)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3648)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2624)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3648)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4672)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4672)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z0); + /* 160..175 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 640)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1664)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1664)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2688)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3712)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2688)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3712)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4736)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4736)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z0); + /* 176..191 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 704)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1728)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1728)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2752)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3776)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2752)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3776)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4800)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4800)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z0); + /* 192..207 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 768)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1792)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1792)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2816)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3840)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2816)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3840)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4864)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4864)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z0); + /* 208..223 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 832)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1856)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1856)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2880)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3904)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2880)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3904)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4928)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4928)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z0); + /* 224..239 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 896)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1920)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1920)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2944)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3968)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2944)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3968)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4992)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4992)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z0); + /* 240..255 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 960)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1984)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1984)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3008)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4032)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3008)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4032)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5056)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5056)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z0); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_mul_vec_7_avx512(sword32* r, const sword32* a, + const sword32* b) +{ + word64 rdi, rsi, rdx, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + rax = (word64)(0x7fe001); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z12 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z12)); + rax = (word64)(0x3802001); + z13 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z13 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z13)); + /* 0..15 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1024)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1024)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2048)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3072)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2048)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3072)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4096)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5120)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4096)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5120)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6144)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6144)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + /* 16..31 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1088)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1088)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2112)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3136)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2112)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3136)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4160)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5184)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4160)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5184)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6208)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6208)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z0); + /* 32..47 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1152)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1152)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2176)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3200)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2176)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3200)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4224)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5248)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4224)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5248)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6272)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6272)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z0); + /* 48..63 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1216)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1216)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2240)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3264)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2240)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3264)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4288)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5312)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4288)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5312)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6336)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6336)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z0); + /* 64..79 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1280)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1280)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2304)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3328)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2304)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3328)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4352)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5376)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4352)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5376)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6400)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6400)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z0); + /* 80..95 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1344)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1344)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2368)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3392)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2368)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3392)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4416)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5440)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4416)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5440)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6464)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6464)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z0); + /* 96..111 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1408)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1408)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2432)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3456)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2432)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3456)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4480)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5504)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4480)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5504)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6528)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6528)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z0); + /* 112..127 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1472)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1472)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2496)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3520)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2496)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3520)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4544)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5568)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4544)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5568)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6592)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6592)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z0); + /* 128..143 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 512)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1536)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 512)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1536)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2560)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3584)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2560)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3584)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4608)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5632)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4608)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5632)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6656)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6656)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z0); + /* 144..159 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1600)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 576)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1600)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2624)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3648)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2624)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3648)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4672)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5696)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4672)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5696)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6720)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6720)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z0); + /* 160..175 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 640)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1664)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 640)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1664)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2688)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3712)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2688)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3712)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4736)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5760)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4736)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5760)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6784)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6784)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z0); + /* 176..191 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 704)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1728)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 704)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1728)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2752)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3776)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2752)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3776)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4800)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5824)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4800)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5824)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6848)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6848)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z0); + /* 192..207 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 768)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1792)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 768)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1792)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2816)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3840)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2816)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3840)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4864)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5888)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4864)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5888)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6912)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6912)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z0); + /* 208..223 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 832)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1856)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 832)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1856)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2880)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3904)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2880)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3904)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4928)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5952)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4928)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5952)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6976)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6976)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z0); + /* 224..239 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 896)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1920)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 896)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1920)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 2944)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3968)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 2944)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3968)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4992)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6016)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4992)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6016)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 7040)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 7040)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z0); + /* 240..255 */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 960)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 1984)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdx, 960)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 1984)); + z1 = _mm512_shuffle_epi32(z0, 0xf5); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z7 = _mm512_shuffle_epi32(z6, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z0 = _mm512_mul_epi32(z0, z6); + z1 = _mm512_mul_epi32(z1, z7); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 3008)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 4032)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 3008)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 4032)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 5056)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 6080)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 5056)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rdx, 6080)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z5 = _mm512_shuffle_epi32(z4, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z11 = _mm512_shuffle_epi32(z10, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z4 = _mm512_mul_epi32(z4, z10); + z5 = _mm512_mul_epi32(z5, z11); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + z0 = _mm512_add_epi64(z0, z4); + z1 = _mm512_add_epi64(z1, z5); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 7104)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rdx, 7104)); + z3 = _mm512_shuffle_epi32(z2, 0xf5); + z9 = _mm512_shuffle_epi32(z8, 0xf5); + z2 = _mm512_mul_epi32(z2, z8); + z3 = _mm512_mul_epi32(z3, z9); + z0 = _mm512_add_epi64(z0, z2); + z1 = _mm512_add_epi64(z1, z3); + /* Mont Reduce */ + z6 = _mm512_mullo_epi32(z0, z13); + z7 = _mm512_mullo_epi32(z1, z13); + z6 = _mm512_mul_epi32(z6, z12); + z7 = _mm512_mul_epi32(z7, z12); + z0 = _mm512_sub_epi32(z0, z6); + z1 = _mm512_sub_epi32(z1, z7); + z0 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_or_si512(z0, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z0); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_decompose_q88_avx512(const sword32* r, sword32* r0, + sword32* r1) +{ + word64 rdi, rsi, rdx, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16; + __mmask16 k1, k2; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + + rax = (word64)(0x17400); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z12 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z12)); + rax = (word64)(0x2e800); + z13 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z13 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z13)); + rax = (word64)(0x3fefd4); + z14 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z14 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z14)); + rax = (word64)(0x2c); + z15 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z15 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z15)); + rax = (word64)(1); + z16 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z16 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z16)); + /* 1/4 vectors */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 128), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 192), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 256), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 320), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 384), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 448), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 512), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 576), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 640), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 704), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 704), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 768), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 832), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 896), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 960), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 768), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 832), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 896), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 960), z11); + /* 2/4 vectors */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1024)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1088)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1152)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1216)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 1024), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 1088), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 1152), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 1216), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 1024), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 1088), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 1152), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 1216), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1280)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1344)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1408)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1472)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 1280), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 1344), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 1408), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 1472), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 1280), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 1344), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 1408), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 1472), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1536)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1600)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1664)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1728)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 1536), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 1600), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 1664), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 1728), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 1536), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 1600), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 1664), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 1728), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1792)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1856)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1920)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1984)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 1792), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 1856), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 1920), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 1984), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 1792), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 1856), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 1920), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 1984), z11); + /* 3/4 vectors */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2048)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2112)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2176)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2240)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 2048), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 2112), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 2176), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 2240), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 2048), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 2112), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 2176), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 2240), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2304)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2368)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2432)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2496)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 2304), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 2368), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 2432), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 2496), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 2304), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 2368), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 2432), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 2496), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2560)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2624)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2688)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2752)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 2560), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 2624), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 2688), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 2752), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 2560), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 2624), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 2688), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 2752), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2816)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2880)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 2944)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3008)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 2816), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 2880), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 2944), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 3008), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 2816), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 2880), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 2944), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 3008), z11); + /* 4/4 vectors */ + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3072)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3136)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3200)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3264)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 3072), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 3136), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 3200), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 3264), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 3072), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 3136), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 3200), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 3264), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3328)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3392)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3456)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3520)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 3328), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 3392), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 3456), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 3520), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 3328), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 3392), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 3456), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 3520), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3584)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3648)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3712)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3776)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 3584), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 3648), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 3712), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 3776), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 3584), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 3648), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 3712), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 3776), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3840)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3904)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 3968)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 4032)); + z8 = _mm512_mullo_epi32(z0, z15); + z9 = _mm512_mullo_epi32(z1, z15); + z10 = _mm512_mullo_epi32(z2, z15); + z11 = _mm512_mullo_epi32(z3, z15); + z8 = _mm512_add_epi32(z8, z14); + z9 = _mm512_add_epi32(z9, z14); + z10 = _mm512_add_epi32(z10, z14); + z11 = _mm512_add_epi32(z11, z14); + z8 = _mm512_srli_epi32(z8, 23); + z9 = _mm512_srli_epi32(z9, 23); + z10 = _mm512_srli_epi32(z10, 23); + z11 = _mm512_srli_epi32(z11, 23); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + k1 = _mm512_cmp_epi32_mask(z8, z15, _MM_CMPINT_EQ); + z4 = _mm512_mask_blend_epi32(k1, z4, _mm512_sub_epi32(z4, z16)); + k1 = _mm512_cmp_epi32_mask(z9, z15, _MM_CMPINT_EQ); + z5 = _mm512_mask_blend_epi32(k1, z5, _mm512_sub_epi32(z5, z16)); + k1 = _mm512_cmp_epi32_mask(z10, z15, _MM_CMPINT_EQ); + z6 = _mm512_mask_blend_epi32(k1, z6, _mm512_sub_epi32(z6, z16)); + k1 = _mm512_cmp_epi32_mask(z11, z15, _MM_CMPINT_EQ); + z7 = _mm512_mask_blend_epi32(k1, z7, _mm512_sub_epi32(z7, z16)); + k2 = _mm512_cmp_epi32_mask(z15, z8, _MM_CMPINT_NLE); + z8 = _mm512_maskz_mov_epi32(k2, z8); + k2 = _mm512_cmp_epi32_mask(z15, z9, _MM_CMPINT_NLE); + z9 = _mm512_maskz_mov_epi32(k2, z9); + k2 = _mm512_cmp_epi32_mask(z15, z10, _MM_CMPINT_NLE); + z10 = _mm512_maskz_mov_epi32(k2, z10); + k2 = _mm512_cmp_epi32_mask(z15, z11, _MM_CMPINT_NLE); + z11 = _mm512_maskz_mov_epi32(k2, z11); + _mm512_storeu_si512((void*)WC_PW(rsi, 3840), z4); + _mm512_storeu_si512((void*)WC_PW(rsi, 3904), z5); + _mm512_storeu_si512((void*)WC_PW(rsi, 3968), z6); + _mm512_storeu_si512((void*)WC_PW(rsi, 4032), z7); + _mm512_storeu_si512((void*)WC_PW(rdx, 3840), z8); + _mm512_storeu_si512((void*)WC_PW(rdx, 3904), z9); + _mm512_storeu_si512((void*)WC_PW(rdx, 3968), z10); + _mm512_storeu_si512((void*)WC_PW(rdx, 4032), z11); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_decompose_q32_avx512(const sword32* r, byte k, + sword32* r0, sword32* r1) +{ + word64 rdi, rsi, rdx, rcx, rax; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), + z8 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), z12, + z13, z14, z15; + + rdi = (word64)(size_t)r; + rsi = (word64)(byte)k; + rdx = (word64)(size_t)r0; + rcx = (word64)(size_t)r1; + + rax = (word64)(0x3ff00); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z12 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z12)); + rax = (word64)(0x7fe00); + z13 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z13 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z13)); + rax = (word64)(0x3feff); + z14 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z14 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z14)); + rax = (word64)(0xf); + z15 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z15 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z15)); +L_mldsa_decompose_q32_avx512_start_256: + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z8 = _mm512_add_epi32(z0, z14); + z9 = _mm512_add_epi32(z1, z14); + z10 = _mm512_add_epi32(z2, z14); + z11 = _mm512_add_epi32(z3, z14); + z8 = _mm512_srli_epi32(z8, 19); + z9 = _mm512_srli_epi32(z9, 19); + z10 = _mm512_srli_epi32(z10, 19); + z11 = _mm512_srli_epi32(z11, 19); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi32(z8, 4); + z1 = _mm512_srli_epi32(z9, 4); + z2 = _mm512_srli_epi32(z10, 4); + z3 = _mm512_srli_epi32(z11, 4); + z4 = _mm512_sub_epi32(z4, z0); + z5 = _mm512_sub_epi32(z5, z1); + z6 = _mm512_sub_epi32(z6, z2); + z7 = _mm512_sub_epi32(z7, z3); + z8 = _mm512_and_si512(z8, z15); + z9 = _mm512_and_si512(z9, z15); + z10 = _mm512_and_si512(z10, z15); + z11 = _mm512_and_si512(z11, z15); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z6); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z7); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z8); + _mm512_storeu_si512((void*)WC_PW(rcx, 64), z9); + _mm512_storeu_si512((void*)WC_PW(rcx, 128), z10); + _mm512_storeu_si512((void*)WC_PW(rcx, 192), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_add_epi32(z0, z14); + z9 = _mm512_add_epi32(z1, z14); + z10 = _mm512_add_epi32(z2, z14); + z11 = _mm512_add_epi32(z3, z14); + z8 = _mm512_srli_epi32(z8, 19); + z9 = _mm512_srli_epi32(z9, 19); + z10 = _mm512_srli_epi32(z10, 19); + z11 = _mm512_srli_epi32(z11, 19); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi32(z8, 4); + z1 = _mm512_srli_epi32(z9, 4); + z2 = _mm512_srli_epi32(z10, 4); + z3 = _mm512_srli_epi32(z11, 4); + z4 = _mm512_sub_epi32(z4, z0); + z5 = _mm512_sub_epi32(z5, z1); + z6 = _mm512_sub_epi32(z6, z2); + z7 = _mm512_sub_epi32(z7, z3); + z8 = _mm512_and_si512(z8, z15); + z9 = _mm512_and_si512(z9, z15); + z10 = _mm512_and_si512(z10, z15); + z11 = _mm512_and_si512(z11, z15); + _mm512_storeu_si512((void*)WC_PW(rdx, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdx, 448), z7); + _mm512_storeu_si512((void*)WC_PW(rcx, 256), z8); + _mm512_storeu_si512((void*)WC_PW(rcx, 320), z9); + _mm512_storeu_si512((void*)WC_PW(rcx, 384), z10); + _mm512_storeu_si512((void*)WC_PW(rcx, 448), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z8 = _mm512_add_epi32(z0, z14); + z9 = _mm512_add_epi32(z1, z14); + z10 = _mm512_add_epi32(z2, z14); + z11 = _mm512_add_epi32(z3, z14); + z8 = _mm512_srli_epi32(z8, 19); + z9 = _mm512_srli_epi32(z9, 19); + z10 = _mm512_srli_epi32(z10, 19); + z11 = _mm512_srli_epi32(z11, 19); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi32(z8, 4); + z1 = _mm512_srli_epi32(z9, 4); + z2 = _mm512_srli_epi32(z10, 4); + z3 = _mm512_srli_epi32(z11, 4); + z4 = _mm512_sub_epi32(z4, z0); + z5 = _mm512_sub_epi32(z5, z1); + z6 = _mm512_sub_epi32(z6, z2); + z7 = _mm512_sub_epi32(z7, z3); + z8 = _mm512_and_si512(z8, z15); + z9 = _mm512_and_si512(z9, z15); + z10 = _mm512_and_si512(z10, z15); + z11 = _mm512_and_si512(z11, z15); + _mm512_storeu_si512((void*)WC_PW(rdx, 512), z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 576), z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 640), z6); + _mm512_storeu_si512((void*)WC_PW(rdx, 704), z7); + _mm512_storeu_si512((void*)WC_PW(rcx, 512), z8); + _mm512_storeu_si512((void*)WC_PW(rcx, 576), z9); + _mm512_storeu_si512((void*)WC_PW(rcx, 640), z10); + _mm512_storeu_si512((void*)WC_PW(rcx, 704), z11); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_add_epi32(z0, z14); + z9 = _mm512_add_epi32(z1, z14); + z10 = _mm512_add_epi32(z2, z14); + z11 = _mm512_add_epi32(z3, z14); + z8 = _mm512_srli_epi32(z8, 19); + z9 = _mm512_srli_epi32(z9, 19); + z10 = _mm512_srli_epi32(z10, 19); + z11 = _mm512_srli_epi32(z11, 19); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z4 = _mm512_sub_epi32(z12, z4); + z5 = _mm512_sub_epi32(z12, z5); + z6 = _mm512_sub_epi32(z12, z6); + z7 = _mm512_sub_epi32(z12, z7); + z4 = _mm512_srli_epi32(z4, 31); + z5 = _mm512_srli_epi32(z5, 31); + z6 = _mm512_srli_epi32(z6, 31); + z7 = _mm512_srli_epi32(z7, 31); + z8 = _mm512_add_epi32(z8, z4); + z9 = _mm512_add_epi32(z9, z5); + z10 = _mm512_add_epi32(z10, z6); + z11 = _mm512_add_epi32(z11, z7); + z4 = _mm512_mullo_epi32(z8, z13); + z5 = _mm512_mullo_epi32(z9, z13); + z6 = _mm512_mullo_epi32(z10, z13); + z7 = _mm512_mullo_epi32(z11, z13); + z4 = _mm512_sub_epi32(z0, z4); + z5 = _mm512_sub_epi32(z1, z5); + z6 = _mm512_sub_epi32(z2, z6); + z7 = _mm512_sub_epi32(z3, z7); + z0 = _mm512_srli_epi32(z8, 4); + z1 = _mm512_srli_epi32(z9, 4); + z2 = _mm512_srli_epi32(z10, 4); + z3 = _mm512_srli_epi32(z11, 4); + z4 = _mm512_sub_epi32(z4, z0); + z5 = _mm512_sub_epi32(z5, z1); + z6 = _mm512_sub_epi32(z6, z2); + z7 = _mm512_sub_epi32(z7, z3); + z8 = _mm512_and_si512(z8, z15); + z9 = _mm512_and_si512(z9, z15); + z10 = _mm512_and_si512(z10, z15); + z11 = _mm512_and_si512(z11, z15); + _mm512_storeu_si512((void*)WC_PW(rdx, 768), z4); + _mm512_storeu_si512((void*)WC_PW(rdx, 832), z5); + _mm512_storeu_si512((void*)WC_PW(rdx, 896), z6); + _mm512_storeu_si512((void*)WC_PW(rdx, 960), z7); + _mm512_storeu_si512((void*)WC_PW(rcx, 768), z8); + _mm512_storeu_si512((void*)WC_PW(rcx, 832), z9); + _mm512_storeu_si512((void*)WC_PW(rcx, 896), z10); + _mm512_storeu_si512((void*)WC_PW(rcx, 960), z11); + rdi = (word64)(rdi + 0x400); + rdx = (word64)(rdx + 0x400); + rcx = (word64)(rcx + 0x400); + rsi = (word64)(rsi - 1); + if ((rsi) != (0)) { + goto L_mldsa_decompose_q32_avx512_start_256; + } +} + +WC_X64I_TARGET("sse2,avx512f") +WOLFSSL_LOCAL int wc_mldsa_vec_check_low_avx512(const sword32* a, byte l, + sword32 hi) +{ + word64 rdi, rsi, rdx, rax = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(); + __mmask16 k1, k3, k2, k4; + word16 zf1; + word16 zf2; + word16 zf3; + word16 zf4; + word16 zf5; + word16 zf6; + word16 zf7; + word16 zf8; + + rdi = (word64)(size_t)a; + rsi = (word64)(byte)l; + rdx = (word64)(word32)hi; + + rdx = (word32)((word32)rdx - 1); + z2 = _mm512_inserti32x4(z2, _mm_cvtsi32_si128((int)(word32)rdx), 0); + rdx = (word32)(0 - (word32)rdx); + z3 = _mm512_inserti32x4(z3, _mm_cvtsi32_si128((int)(word32)rdx), 0); + z2 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z2)); + z3 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z3)); +L_mldsa_vec_check_low_avx512_start_256: + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + k1 = _mm512_cmp_epi32_mask(z0, z2, _MM_CMPINT_NLE); + k3 = _mm512_cmp_epi32_mask(z1, z2, _MM_CMPINT_NLE); + k2 = _mm512_cmp_epi32_mask(z3, z0, _MM_CMPINT_NLE); + k4 = _mm512_cmp_epi32_mask(z3, z1, _MM_CMPINT_NLE); + k1 = _kor_mask16(k1, k3); + k2 = _kor_mask16(k2, k4); + k1 = _kor_mask16(k1, k2); + zf1 = k1; + rax = (word64)(0); + if ((zf1) != (0)) { + goto L_mldsa_vec_check_low_avx512_done; + } + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + k1 = _mm512_cmp_epi32_mask(z0, z2, _MM_CMPINT_NLE); + k3 = _mm512_cmp_epi32_mask(z1, z2, _MM_CMPINT_NLE); + k2 = _mm512_cmp_epi32_mask(z3, z0, _MM_CMPINT_NLE); + k4 = _mm512_cmp_epi32_mask(z3, z1, _MM_CMPINT_NLE); + k1 = _kor_mask16(k1, k3); + k2 = _kor_mask16(k2, k4); + k1 = _kor_mask16(k1, k2); + zf2 = k1; + rax = (word64)(0); + if ((zf2) != (0)) { + goto L_mldsa_vec_check_low_avx512_done; + } + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + k1 = _mm512_cmp_epi32_mask(z0, z2, _MM_CMPINT_NLE); + k3 = _mm512_cmp_epi32_mask(z1, z2, _MM_CMPINT_NLE); + k2 = _mm512_cmp_epi32_mask(z3, z0, _MM_CMPINT_NLE); + k4 = _mm512_cmp_epi32_mask(z3, z1, _MM_CMPINT_NLE); + k1 = _kor_mask16(k1, k3); + k2 = _kor_mask16(k2, k4); + k1 = _kor_mask16(k1, k2); + zf3 = k1; + rax = (word64)(0); + if ((zf3) != (0)) { + goto L_mldsa_vec_check_low_avx512_done; + } + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + k1 = _mm512_cmp_epi32_mask(z0, z2, _MM_CMPINT_NLE); + k3 = _mm512_cmp_epi32_mask(z1, z2, _MM_CMPINT_NLE); + k2 = _mm512_cmp_epi32_mask(z3, z0, _MM_CMPINT_NLE); + k4 = _mm512_cmp_epi32_mask(z3, z1, _MM_CMPINT_NLE); + k1 = _kor_mask16(k1, k3); + k2 = _kor_mask16(k2, k4); + k1 = _kor_mask16(k1, k2); + zf4 = k1; + rax = (word64)(0); + if ((zf4) != (0)) { + goto L_mldsa_vec_check_low_avx512_done; + } + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + k1 = _mm512_cmp_epi32_mask(z0, z2, _MM_CMPINT_NLE); + k3 = _mm512_cmp_epi32_mask(z1, z2, _MM_CMPINT_NLE); + k2 = _mm512_cmp_epi32_mask(z3, z0, _MM_CMPINT_NLE); + k4 = _mm512_cmp_epi32_mask(z3, z1, _MM_CMPINT_NLE); + k1 = _kor_mask16(k1, k3); + k2 = _kor_mask16(k2, k4); + k1 = _kor_mask16(k1, k2); + zf5 = k1; + rax = (word64)(0); + if ((zf5) != (0)) { + goto L_mldsa_vec_check_low_avx512_done; + } + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + k1 = _mm512_cmp_epi32_mask(z0, z2, _MM_CMPINT_NLE); + k3 = _mm512_cmp_epi32_mask(z1, z2, _MM_CMPINT_NLE); + k2 = _mm512_cmp_epi32_mask(z3, z0, _MM_CMPINT_NLE); + k4 = _mm512_cmp_epi32_mask(z3, z1, _MM_CMPINT_NLE); + k1 = _kor_mask16(k1, k3); + k2 = _kor_mask16(k2, k4); + k1 = _kor_mask16(k1, k2); + zf6 = k1; + rax = (word64)(0); + if ((zf6) != (0)) { + goto L_mldsa_vec_check_low_avx512_done; + } + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + k1 = _mm512_cmp_epi32_mask(z0, z2, _MM_CMPINT_NLE); + k3 = _mm512_cmp_epi32_mask(z1, z2, _MM_CMPINT_NLE); + k2 = _mm512_cmp_epi32_mask(z3, z0, _MM_CMPINT_NLE); + k4 = _mm512_cmp_epi32_mask(z3, z1, _MM_CMPINT_NLE); + k1 = _kor_mask16(k1, k3); + k2 = _kor_mask16(k2, k4); + k1 = _kor_mask16(k1, k2); + zf7 = k1; + rax = (word64)(0); + if ((zf7) != (0)) { + goto L_mldsa_vec_check_low_avx512_done; + } + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + k1 = _mm512_cmp_epi32_mask(z0, z2, _MM_CMPINT_NLE); + k3 = _mm512_cmp_epi32_mask(z1, z2, _MM_CMPINT_NLE); + k2 = _mm512_cmp_epi32_mask(z3, z0, _MM_CMPINT_NLE); + k4 = _mm512_cmp_epi32_mask(z3, z1, _MM_CMPINT_NLE); + k1 = _kor_mask16(k1, k3); + k2 = _kor_mask16(k2, k4); + k1 = _kor_mask16(k1, k2); + zf8 = k1; + rax = (word64)(0); + if ((zf8) != (0)) { + goto L_mldsa_vec_check_low_avx512_done; + } + rdi = (word64)(rdi + 0x400); + rsi = (word64)(rsi - 1); + if ((rsi) != (0)) { + goto L_mldsa_vec_check_low_avx512_start_256; + } + rax = (word64)(1); +L_mldsa_vec_check_low_avx512_done: + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void wc_mldsa_poly_add_avx512(sword32* r, const sword32* a) +{ + word64 rdi, rsi; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z0 = _mm512_add_epi32(z0, z8); + z1 = _mm512_add_epi32(z1, z9); + z2 = _mm512_add_epi32(z2, z10); + z3 = _mm512_add_epi32(z3, z11); + z4 = _mm512_add_epi32(z4, z12); + z5 = _mm512_add_epi32(z5, z13); + z6 = _mm512_add_epi32(z6, z14); + z7 = _mm512_add_epi32(z7, z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rsi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rsi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rsi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rsi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rsi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rsi, 960)); + z0 = _mm512_add_epi32(z0, z8); + z1 = _mm512_add_epi32(z1, z9); + z2 = _mm512_add_epi32(z2, z10); + z3 = _mm512_add_epi32(z3, z11); + z4 = _mm512_add_epi32(z4, z12); + z5 = _mm512_add_epi32(z5, z13); + z6 = _mm512_add_epi32(z6, z14); + z7 = _mm512_add_epi32(z7, z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z7); +} + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void wc_mldsa_poly_sub_avx512(sword32* r, const sword32* a) +{ + word64 rdi, rsi; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z0 = _mm512_sub_epi32(z0, z8); + z1 = _mm512_sub_epi32(z1, z9); + z2 = _mm512_sub_epi32(z2, z10); + z3 = _mm512_sub_epi32(z3, z11); + z4 = _mm512_sub_epi32(z4, z12); + z5 = _mm512_sub_epi32(z5, z13); + z6 = _mm512_sub_epi32(z6, z14); + z7 = _mm512_sub_epi32(z7, z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 512)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 576)); + z10 = _mm512_loadu_si512((const void*)WC_PR(rsi, 640)); + z11 = _mm512_loadu_si512((const void*)WC_PR(rsi, 704)); + z12 = _mm512_loadu_si512((const void*)WC_PR(rsi, 768)); + z13 = _mm512_loadu_si512((const void*)WC_PR(rsi, 832)); + z14 = _mm512_loadu_si512((const void*)WC_PR(rsi, 896)); + z15 = _mm512_loadu_si512((const void*)WC_PR(rsi, 960)); + z0 = _mm512_sub_epi32(z0, z8); + z1 = _mm512_sub_epi32(z1, z9); + z2 = _mm512_sub_epi32(z2, z10); + z3 = _mm512_sub_epi32(z3, z11); + z4 = _mm512_sub_epi32(z4, z12); + z5 = _mm512_sub_epi32(z5, z13); + z6 = _mm512_sub_epi32(z6, z14); + z7 = _mm512_sub_epi32(z7, z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z7); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void wc_mldsa_poly_make_pos_avx512(sword32* a) +{ + word64 rdi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16; + + rdi = (word64)(size_t)a; + + rdx = (word64)(0x7fe001); + z16 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z16 = _mm512_broadcastd_epi32(_mm512_castsi512_si128(z16)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_srai_epi32(z0, 31); + z9 = _mm512_srai_epi32(z1, 31); + z10 = _mm512_srai_epi32(z2, 31); + z11 = _mm512_srai_epi32(z3, 31); + z12 = _mm512_srai_epi32(z4, 31); + z13 = _mm512_srai_epi32(z5, 31); + z14 = _mm512_srai_epi32(z6, 31); + z15 = _mm512_srai_epi32(z7, 31); + z8 = _mm512_and_si512(z8, z16); + z9 = _mm512_and_si512(z9, z16); + z10 = _mm512_and_si512(z10, z16); + z11 = _mm512_and_si512(z11, z16); + z12 = _mm512_and_si512(z12, z16); + z13 = _mm512_and_si512(z13, z16); + z14 = _mm512_and_si512(z14, z16); + z15 = _mm512_and_si512(z15, z16); + z0 = _mm512_add_epi32(z0, z8); + z1 = _mm512_add_epi32(z1, z9); + z2 = _mm512_add_epi32(z2, z10); + z3 = _mm512_add_epi32(z3, z11); + z4 = _mm512_add_epi32(z4, z12); + z5 = _mm512_add_epi32(z5, z13); + z6 = _mm512_add_epi32(z6, z14); + z7 = _mm512_add_epi32(z7, z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_srai_epi32(z0, 31); + z9 = _mm512_srai_epi32(z1, 31); + z10 = _mm512_srai_epi32(z2, 31); + z11 = _mm512_srai_epi32(z3, 31); + z12 = _mm512_srai_epi32(z4, 31); + z13 = _mm512_srai_epi32(z5, 31); + z14 = _mm512_srai_epi32(z6, 31); + z15 = _mm512_srai_epi32(z7, 31); + z8 = _mm512_and_si512(z8, z16); + z9 = _mm512_and_si512(z9, z16); + z10 = _mm512_and_si512(z10, z16); + z11 = _mm512_and_si512(z11, z16); + z12 = _mm512_and_si512(z12, z16); + z13 = _mm512_and_si512(z13, z16); + z14 = _mm512_and_si512(z14, z16); + z15 = _mm512_and_si512(z15, z16); + z0 = _mm512_add_epi32(z0, z8); + z1 = _mm512_add_epi32(z1, z9); + z2 = _mm512_add_epi32(z2, z10); + z3 = _mm512_add_epi32(z3, z11); + z4 = _mm512_add_epi32(z4, z12); + z5 = _mm512_add_epi32(z5, z13); + z6 = _mm512_add_epi32(z6, z14); + z7 = _mm512_add_epi32(z7, z15); + _mm512_storeu_si512((void*)WC_PW(rdi, 512), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 576), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 640), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 704), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 768), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 832), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 896), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 960), z7); +} + +#endif /* HAVE_INTEL_AVX512 */ +#endif /* WOLFSSL_HAVE_MLDSA */ +#ifdef WOLFSSL_HAVE_MLDSA +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ +#ifdef HAVE_INTEL_AVX512_VBMI +XALIGNED(16) static const word32 L_mldsa_encode_w1_88_avx512_vbmi_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000006, 0x0000000c, 0x00000012, + 0x00000000, 0x00000006, 0x0000000c, 0x00000012, + 0x00000000, 0x00000006, 0x0000000c, 0x00000012, + 0x00000000, 0x00000006, 0x0000000c, 0x00000012, +}; + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_avx512_vbmi_idx_a[] + WC_X64I_UNUSED = { + 0x00, 0x05, 0x0a, 0x10, 0x15, 0x1a, 0x20, 0x25, + 0x2a, 0x30, 0x35, 0x3a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +XALIGNED(16) static const byte L_mldsa_encode_w1_88_avx512_vbmi_idx_b[] + WC_X64I_UNUSED = { + 0x04, 0x09, 0x0e, 0x14, 0x19, 0x1e, 0x24, 0x29, + 0x2e, 0x34, 0x39, 0x3e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +WC_X64I_TARGET("avx512f,avx512vbmi") +WOLFSSL_LOCAL void wc_mldsa_encode_w1_88_avx512_vbmi(const sword32* w1, + byte* w1e) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4; + __mmask16 k1; + + rdi = (word64)(size_t)w1; + rsi = (word64)(size_t)w1e; + + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_w1_88_avx512_vbmi_vs, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_w1_88_avx512_vbmi_idx_a, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_w1_88_avx512_vbmi_idx_b, 0)); + rdx = (word32)(7); + k1 = (__mmask16)(word32)rdx; + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 0), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 12), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 24), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 36), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 48), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 60), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 72), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 84), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 96), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 108), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 120), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 132), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 144), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 156), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 168), k1, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z0 = _mm512_sllv_epi32(z0, z4); + z1 = _mm512_permutexvar_epi8(z3, z0); + z0 = _mm512_permutexvar_epi8(z2, z0); + z0 = _mm512_or_si512(z0, z1); + _mm512_mask_storeu_epi32(WC_PW(rsi, 180), k1, z0); +} + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_avx512_vbmi_half_m1[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_avx512_vbmi_half[] + WC_X64I_UNUSED = { + 0x00001000, 0x00001000, 0x00001000, 0x00001000, + 0x00001000, 0x00001000, 0x00001000, 0x00001000, + 0x00001000, 0x00001000, 0x00001000, 0x00001000, + 0x00001000, 0x00001000, 0x00001000, 0x00001000, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_avx512_vbmi_t0_idx0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x05, 0x06, 0x09, 0x0d, 0x0e, 0x11, + 0x12, 0x15, 0x19, 0x1a, 0x1d, 0x20, 0x21, 0x25, + 0x26, 0x29, 0x2d, 0x2e, 0x31, 0x32, 0x35, 0x39, + 0x3a, 0x3d, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_avx512_vbmi_t0_idx1[] + WC_X64I_UNUSED = { + 0x03, 0x04, 0x03, 0x08, 0x0c, 0x03, 0x10, 0x03, + 0x14, 0x18, 0x03, 0x1c, 0x03, 0x03, 0x24, 0x03, + 0x28, 0x2c, 0x03, 0x30, 0x03, 0x34, 0x38, 0x03, + 0x3c, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_avx512_vbmi_t0_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000005, 0x00000002, 0x00000007, + 0x00000004, 0x00000001, 0x00000006, 0x00000003, + 0x00000000, 0x00000005, 0x00000002, 0x00000007, + 0x00000004, 0x00000001, 0x00000006, 0x00000003, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_avx512_vbmi_t1_idx0[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x05, 0x09, 0x0d, 0x10, 0x11, 0x15, + 0x19, 0x1d, 0x20, 0x21, 0x25, 0x29, 0x2d, 0x30, + 0x31, 0x35, 0x39, 0x3d, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, +}; + +XALIGNED(16) static const byte L_mldsa_encode_t0_t1_avx512_vbmi_t1_idx1[] + WC_X64I_UNUSED = { + 0x03, 0x04, 0x08, 0x0c, 0x03, 0x03, 0x14, 0x18, + 0x1c, 0x03, 0x03, 0x24, 0x28, 0x2c, 0x03, 0x03, + 0x34, 0x38, 0x3c, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, +}; + +XALIGNED(16) static const word32 L_mldsa_encode_t0_t1_avx512_vbmi_t1_vs[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000002, 0x00000004, 0x00000006, + 0x00000000, 0x00000002, 0x00000004, 0x00000006, + 0x00000000, 0x00000002, 0x00000004, 0x00000006, + 0x00000000, 0x00000002, 0x00000004, 0x00000006, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi") +WOLFSSL_LOCAL void wc_mldsa_encode_t0_t1_avx512_vbmi(const sword32* t, byte* t0, + byte* t1) +{ + word64 rdi, rsi, rdx, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11; + __mmask64 k1, k2; + + rdi = (word64)(size_t)t; + rsi = (word64)(size_t)t0; + rdx = (word64)(size_t)t1; + + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_t0_t1_avx512_vbmi_half_m1, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_t0_t1_avx512_vbmi_half, 0)); + z6 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_t0_t1_avx512_vbmi_t0_vs, 0)); + z7 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_t0_t1_avx512_vbmi_t1_vs, 0)); + z8 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_t0_t1_avx512_vbmi_t0_idx0, 0)); + z9 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_t0_t1_avx512_vbmi_t0_idx1, 0)); + z10 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_t0_t1_avx512_vbmi_t1_idx0, 0)); + z11 = _mm512_loadu_si512((const void*)WC_PR( + L_mldsa_encode_t0_t1_avx512_vbmi_t1_idx1, 0)); + rax = (word32)(0x3ffffff); + k1 = (__mmask32)(word32)rax; + rax = (word32)(0xfffff); + k2 = (__mmask32)(word32)rax; + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 0), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 0), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 26), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 20), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 52), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 40), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 78), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 60), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 104), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 80), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 130), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 100), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 156), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 120), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 182), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 140), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 208), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 160), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 234), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 180), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 260), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 200), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 286), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 220), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 312), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 240), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 338), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 260), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 364), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 280), k2, z0); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z2 = _mm512_add_epi32(z0, z4); + z2 = _mm512_srli_epi32(z2, 13); + z1 = _mm512_slli_epi32(z2, 13); + z1 = _mm512_sub_epi32(z0, z1); + z1 = _mm512_sub_epi32(z5, z1); + z1 = _mm512_sllv_epi32(z1, z6); + z0 = _mm512_permutexvar_epi8(z8, z1); + z3 = _mm512_permutexvar_epi8(z9, z1); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rsi, 390), k1, z0); + z2 = _mm512_sllv_epi32(z2, z7); + z0 = _mm512_permutexvar_epi8(z10, z2); + z3 = _mm512_permutexvar_epi8(z11, z2); + z0 = _mm512_or_si512(z0, z3); + _mm512_mask_storeu_epi8(WC_PW(rdx, 300), k2, z0); +} + +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* WOLFSSL_HAVE_MLDSA */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */ diff --git a/wolfcrypt/src/wc_mlkem_intrin.c b/wolfcrypt/src/wc_mlkem_intrin.c new file mode 100644 index 00000000000..54c07181821 --- /dev/null +++ b/wolfcrypt/src/wc_mlkem_intrin.c @@ -0,0 +1,39467 @@ +/* wc_mlkem_intrin.c */ +/* + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_WC_MLKEM_INTRIN_C + +#include +#include +#ifdef WOLFSSL_X86_64_BUILD + +#ifdef _MSC_VER + /* _umul128, __popcnt and _BitScanReverse64 live here. */ + #include +#endif +#include +#include + +#if !defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER) + #error "Generated intrinsics need GCC, Clang or MSVC." +#endif + +#ifdef _MSC_VER + /* MSVC has no per-function ISA attribute and needs none: it accepts + * any intrinsic whatever /arch says and emits the instruction, so + * the SSE2, AVX2 and AVX-512 variants below all compile from one + * translation unit built at the default /arch. That is the same + * property the target attribute buys on GCC, arrived at from the + * other direction - which is why the wide-ISA guard further down + * still applies: /arch:AVX2 lets MSVC promote the SSE2 variants to + * VEX encodings, and one picked for a pre-AVX CPU would trap. */ + #define WC_X64I_TARGET(isa) /* null expansion */ + #define WC_X64I_UNUSED +#else +/* Select the ISA for one function only, so variants for different ISAs + * can live in one translation unit compiled at baseline -march and + * still be picked at run time by the cpuid dispatch. */ +#define WC_X64I_TARGET(isa) __attribute__((target(isa))) +#define WC_X64I_UNUSED __attribute__((unused)) +#endif + +/* 64x64 -> 128 multiply. GCC and Clang have a 128-bit integer type; + * MSVC has an intrinsic that returns the halves separately. Written as + * one macro so the generator emits the same statement either way. The + * destination may name the same variable as an operand - MUL writes + * RDX:RAX and reads RAX - so both forms read the operands before + * assigning. */ +#ifdef _MSC_VER + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + word64 wc_h_; \ + word64 wc_l_ = (word64)_umul128((a), (b), &wc_h_); \ + (lo) = wc_l_; (hi) = wc_h_; \ + } while (0) +#else + #define WC_X64I_MUL128(lo, hi, a, b) \ + do { \ + __uint128_t wc_p_ = (__uint128_t)(a) * (b); \ + (lo) = (word64)wc_p_; \ + (hi) = (word64)(wc_p_ >> 64); \ + } while (0) +#endif + +/* Byte reversal, and the 128/64 unsigned divide SP's division step + * needs. The divide has no portable spelling at all: GCC and Clang + * take it through __uint128_t, MSVC has _udiv128. Neither traps the + * quotient overflow that DIV raises - the callers keep the divisor + * above the high half, which is what makes the instruction safe there + * too. */ +#ifdef _MSC_VER + #define WC_X64I_BSWAP64(x) ((word64)_byteswap_uint64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + word64 wc_r_; \ + word64 wc_q_ = _udiv128((hi), (lo), (d), &wc_r_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#else + #define WC_X64I_BSWAP64(x) ((word64)__builtin_bswap64((x))) + #define WC_X64I_DIV128(q, r, hi, lo, d) \ + do { \ + __uint128_t wc_n_ = ((__uint128_t)(hi) << 64) | (lo); \ + word64 wc_d_ = (d); \ + word64 wc_q_ = (word64)(wc_n_ / wc_d_); \ + word64 wc_r_ = (word64)(wc_n_ % wc_d_); \ + (q) = wc_q_; (r) = wc_r_; \ + } while (0) +#endif + +/* Bit counting: the two compilers spell these differently and MSVC's + * scan intrinsics report through an out-parameter. BSR is only reached + * with a non-zero source (the generator says so where it emits it), so + * the zero case need not be defined here either. */ +#ifdef _MSC_VER + #define WC_X64I_POPCNT32(x) ((word32)__popcnt((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) ((word64)__popcnt64((unsigned __int64)(x))) + #define WC_X64I_BSR64(x) wc_x64i_bsr64(x) + static WC_X64I_UNUSED word64 wc_x64i_bsr64(word64 x) + { + unsigned long i; + _BitScanReverse64(&i, (unsigned __int64)x); + return (word64)i; + } +#else + #define WC_X64I_POPCNT32(x) \ + ((word32)__builtin_popcount((unsigned int)(x))) + #define WC_X64I_POPCNT64(x) \ + ((word64)__builtin_popcountll((unsigned long long)(x))) + #define WC_X64I_BSR64(x) \ + ((word64)(63 - __builtin_clzll((unsigned long long)(x)))) +#endif + +/* The attribute ADDS to the command line rather than replacing it, so a + * build that enables a wider ISA globally lets EVERY routine here use + * it - and one the cpuid dispatch picked for, say, an AVX2 CPU would + * then execute AVX-512 encodings and trap. The assembly this replaces + * cannot be promoted that way, so refuse rather than silently build it. + * Narrowing the attributes instead is not possible: an attribute set + * smaller than the command line stops the always_inline intrinsics in + * the headers from inlining at all. Define the escape hatch only for a + * build that will run on the machine it was compiled for. */ +#if (defined(__AVX2__) || defined(__AVX512F__)) && \ + !defined(WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA) + /* Override with WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA only for a + * binary that will run on the machine it was built on. */ + #error "wide -march breaks the cpuid dispatch - see above" +#endif + +/* GCC gained _mm256_zextsi128_si256 in 10.x; earlier releases express the + * same "128-bit VEX write zeroes the upper lanes" rule this way. */ +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 10) + #define _mm256_zextsi128_si256(a) \ + _mm256_insertf128_si256(_mm256_setzero_si256(), (a), 0) +#endif + +/* A register holding an address is modelled as a word64, so a memory + * operand is that value cast back to a byte pointer plus the + * displacement. Loads go through the const forms; the element type of + * every object addressed this way matches the access width, which is + * what keeps the accesses free of aliasing violations. */ +#define WC_PR(b, o) ((const byte*)(size_t)(b) + (o)) +#define WC_PW(b, o) ((byte*)(size_t)(b) + (o)) +#define WC_L8(b, o) (*(const byte*)WC_PR(b, o)) +#define WC_L16(b, o) (*(const word16*)WC_PR(b, o)) +#define WC_L32(b, o) (*(const word32*)WC_PR(b, o)) +#define WC_L64(b, o) (*(const word64*)WC_PR(b, o)) +#define WC_S8(b, o) (*(byte*)WC_PW(b, o)) +#define WC_S16(b, o) (*(word16*)WC_PW(b, o)) +#define WC_S32(b, o) (*(word32*)WC_PW(b, o)) +#define WC_S64(b, o) (*(word64*)WC_PW(b, o)) + +/* A stack frame slot is written at one width and read at another, so + * the storage's declared type has to admit both: a union member is the + * access the aliasing rules allow. */ +typedef union { + byte b[8]; + word16 w16[4]; + word32 w32[2]; + word64 w64; +} WC_X64I_SLOT; + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ +#ifndef NO_VAES_SUPPORT +#ifndef HAVE_INTEL_VAES +#define HAVE_INTEL_VAES +#endif /* HAVE_INTEL_VAES */ +#endif /* NO_VAES_SUPPORT */ +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#endif /* NO_AVX512_SUPPORT */ +#ifndef HAVE_INTEL_SSSE3 +#define HAVE_INTEL_SSSE3 +#endif /* HAVE_INTEL_SSSE3 */ +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_IFMA_SUPPORT +#ifndef HAVE_INTEL_AVX512_IFMA +#define HAVE_INTEL_AVX512_IFMA +#endif /* HAVE_INTEL_AVX512_IFMA */ +#endif /* NO_AVX512_IFMA_SUPPORT */ +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ + +#ifdef WOLFSSL_HAVE_MLKEM +#ifdef HAVE_INTEL_AVX2 +extern WOLFSSL_LOCAL void mlkem_keygen_avx2(sword16* priv, sword16* pub, + sword16* e, const sword16* a, int kp); +extern WOLFSSL_LOCAL void mlkem_encapsulate_avx2(const sword16* pub, + sword16* bp, sword16* vp, const sword16* at, sword16* sp, const sword16* ep, + const sword16* epp, const sword16* m, int kp); +extern WOLFSSL_LOCAL void mlkem_decapsulate_avx2(const sword16* priv, + sword16* mp, sword16* bp, const sword16* vp, int kp); +extern WOLFSSL_LOCAL void mlkem_csubq_avx2(sword16* p); +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx2(sword16* p, + unsigned int len, const byte* r, unsigned int rLen); +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx2(sword16* p, + unsigned int len, const byte* r, unsigned int rLen); +extern WOLFSSL_LOCAL void mlkem_cbd_eta3_avx2(sword16* p, const byte* r); +extern WOLFSSL_LOCAL void mlkem_cbd_eta2_avx2(sword16* p, const byte* r); +extern WOLFSSL_LOCAL void mlkem_compress_10_avx2(byte* r, const sword16* p, + int n); +extern WOLFSSL_LOCAL void mlkem_decompress_10_avx2(sword16* p, const byte* r, + int n); +extern WOLFSSL_LOCAL void mlkem_compress_11_avx2(byte* r, const sword16* p, + int n); +extern WOLFSSL_LOCAL void mlkem_decompress_11_avx2(sword16* p, const byte* r, + int n); +extern WOLFSSL_LOCAL void mlkem_compress_4_avx2(byte* b, const sword16* p); +extern WOLFSSL_LOCAL void mlkem_decompress_4_avx2(sword16* p, const byte* r); +extern WOLFSSL_LOCAL void mlkem_compress_5_avx2(byte* b, const sword16* p); +extern WOLFSSL_LOCAL void mlkem_decompress_5_avx2(sword16* p, const byte* r); +extern WOLFSSL_LOCAL void mlkem_from_msg_avx2(sword16* p, const byte* m); +extern WOLFSSL_LOCAL void mlkem_to_msg_avx2(byte* m, sword16* p); +extern WOLFSSL_LOCAL void mlkem_from_bytes_avx2(sword16* p, const byte* b); +extern WOLFSSL_LOCAL void mlkem_to_bytes_avx2(byte* b, sword16* p); +extern WOLFSSL_LOCAL int mlkem_cmp_avx2(const byte* a, const byte* b, int sz); +extern WOLFSSL_LOCAL void mlkem_redistribute_21_rand_avx2(const word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +extern WOLFSSL_LOCAL void mlkem_redistribute_17_rand_avx2(const word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +extern WOLFSSL_LOCAL void mlkem_redistribute_16_rand_avx2(const word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +extern WOLFSSL_LOCAL void mlkem_redistribute_8_rand_avx2(const word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +#endif +#ifdef HAVE_INTEL_AVX512 +extern WOLFSSL_LOCAL void mlkem_keygen_avx512(sword16* priv, sword16* pub, + sword16* e, const sword16* a, int kp); +extern WOLFSSL_LOCAL void mlkem_encapsulate_avx512(const sword16* pub, + sword16* bp, sword16* vp, const sword16* at, sword16* sp, const sword16* ep, + const sword16* epp, const sword16* m, int kp); +extern WOLFSSL_LOCAL void mlkem_decapsulate_avx512(const sword16* priv, + sword16* mp, sword16* bp, const sword16* vp, int kp); +#ifdef WOLFSSL_MLKEM_ASM_TEST +extern WOLFSSL_LOCAL void mlkem_basemul_x2_avx512(sword16* r, const sword16* a, + const sword16* b); +extern WOLFSSL_LOCAL void mlkem_tomont_x2_avx512(sword16* v); +extern WOLFSSL_LOCAL void mlkem_adderr_x2_avx512(sword16* r, const sword16* e); +extern WOLFSSL_LOCAL void mlkem_suberr_x2_avx512(sword16* r, const sword16* e); +extern WOLFSSL_LOCAL void mlkem_ptacc_x2_avx512(sword16* r, const sword16* a, + const sword16* b, int kp); +#endif +extern WOLFSSL_LOCAL void mlkem_cbd_eta2_avx512(sword16* p, const byte* r); +extern WOLFSSL_LOCAL void mlkem_cbd_eta3_avx512(sword16* p, const byte* r); +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx512(sword16* p, + unsigned int len, const byte* r, unsigned int rLen); +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx512(sword16* p, + unsigned int len, const byte* r, unsigned int rLen); +extern WOLFSSL_LOCAL void mlkem_redistribute_8_rand_avx512(const word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +extern WOLFSSL_LOCAL void mlkem_redistribute_16_rand_avx512(const word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +extern WOLFSSL_LOCAL void mlkem_redistribute_17_rand_avx512(const word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +extern WOLFSSL_LOCAL void mlkem_redistribute_21_rand_avx512(const word64* s, + byte* r0, byte* r1, byte* r2, byte* r3); +extern WOLFSSL_LOCAL void mlkem_redistribute_8_rand_x8_avx512(const word64* s, + byte* out, word32 stride); +extern WOLFSSL_LOCAL void mlkem_redistribute_16_rand_x8_avx512(const word64* s, + byte* out, word32 stride); +extern WOLFSSL_LOCAL void mlkem_redistribute_17_rand_x8_avx512(const word64* s, + byte* out, word32 stride); +extern WOLFSSL_LOCAL void mlkem_redistribute_21_rand_x8_avx512(const word64* s, + byte* out, word32 stride); +extern WOLFSSL_LOCAL void mlkem_csubq_avx512(sword16* p); +extern WOLFSSL_LOCAL int mlkem_cmp_avx512(const byte* a, const byte* b, int sz); +extern WOLFSSL_LOCAL void mlkem_from_bytes_avx512(sword16* p, const byte* b); +extern WOLFSSL_LOCAL void mlkem_to_bytes_avx512(byte* b, sword16* p); +extern WOLFSSL_LOCAL void mlkem_from_msg_avx512(sword16* p, const byte* msg); +extern WOLFSSL_LOCAL void mlkem_to_msg_avx512(byte* m, sword16* p); +extern WOLFSSL_LOCAL void mlkem_compress_4_avx512(byte* b, const sword16* p); +extern WOLFSSL_LOCAL void mlkem_decompress_4_avx512(sword16* p, const byte* b); +extern WOLFSSL_LOCAL void mlkem_decompress_5_avx512(sword16* p, const byte* b); +extern WOLFSSL_LOCAL void mlkem_decompress_10_avx512(sword16* p, const byte* r, + int n); +extern WOLFSSL_LOCAL void mlkem_decompress_11_avx512(sword16* p, const byte* r, + int n); +extern WOLFSSL_LOCAL void mlkem_compress_5_avx512(byte* b, const sword16* p); +extern WOLFSSL_LOCAL void mlkem_compress_10_avx512(byte* r, const sword16* p, + int n); +extern WOLFSSL_LOCAL void mlkem_compress_11_avx512(byte* r, const sword16* p, + int n); +#endif +#ifdef HAVE_INTEL_AVX512_VBMI2 +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx512_vbmi2(sword16* p, + unsigned int len, const byte* r, unsigned int rLen); +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx512_vbmi2(sword16* p, + unsigned int len, const byte* r, unsigned int rLen); +#endif +#ifdef HAVE_INTEL_AVX512_VBMI +extern WOLFSSL_LOCAL void mlkem_from_bytes_avx512_vbmi(sword16* p, + const byte* b); +extern WOLFSSL_LOCAL void mlkem_to_bytes_avx512_vbmi(byte* b, sword16* p); +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx512_vbmi(sword16* p, + unsigned int len, const byte* r, unsigned int rLen); +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx512_vbmi(sword16* p, + unsigned int len, const byte* r, unsigned int rLen); +extern WOLFSSL_LOCAL void mlkem_compress_4_avx512_vbmi(byte* b, + const sword16* p); +extern WOLFSSL_LOCAL void mlkem_compress_5_avx512_vbmi(byte* b, + const sword16* p); +extern WOLFSSL_LOCAL void mlkem_compress_10_avx512_vbmi(byte* r, + const sword16* p, int n); +extern WOLFSSL_LOCAL void mlkem_decompress_10_avx512_vbmi(sword16* p, + const byte* r, int n); +extern WOLFSSL_LOCAL void mlkem_decompress_11_avx512_vbmi(sword16* p, + const byte* r, int n); +#ifdef HAVE_INTEL_AVX512_VBMI2 +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx512_vbmi_vbmi2( + sword16* p, unsigned int len, const byte* r, unsigned int rLen); +extern WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx512_vbmi_vbmi2( + sword16* p, unsigned int len, const byte* r, unsigned int rLen); + +#endif +#endif +#endif +#ifdef WOLFSSL_HAVE_MLKEM +#ifdef HAVE_INTEL_AVX2 +XALIGNED(16) static const word16 mlkem_q[] WC_X64I_UNUSED = { + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, +}; + +XALIGNED(16) static const word16 mlkem_qinv[] WC_X64I_UNUSED = { + 0xf301, 0xf301, 0xf301, 0xf301, 0xf301, 0xf301, 0xf301, 0xf301, + 0xf301, 0xf301, 0xf301, 0xf301, 0xf301, 0xf301, 0xf301, 0xf301, +}; + +XALIGNED(16) static const word16 mlkem_f[] WC_X64I_UNUSED = { + 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, + 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, +}; + +XALIGNED(16) static const word16 mlkem_f_qinv[] WC_X64I_UNUSED = { + 0x5049, 0x5049, 0x5049, 0x5049, 0x5049, 0x5049, 0x5049, 0x5049, + 0x5049, 0x5049, 0x5049, 0x5049, 0x5049, 0x5049, 0x5049, 0x5049, +}; + +XALIGNED(16) static const word16 mlkem_v[] WC_X64I_UNUSED = { + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, +}; + +XALIGNED(16) static const word16 L_mlkem_avx2_zetas[] WC_X64I_UNUSED = { + 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, + 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, + 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, + 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, + 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, + 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, + 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, + 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, + 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, + 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, + 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, + 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, + 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, + 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, + 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, + 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, + 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, + 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, + 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, + 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, + 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, + 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, + 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, + 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, + 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, + 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, + 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, + 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, + 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, + 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, + 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, + 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, + 0x023d, 0x023d, 0x023d, 0x023d, 0x023d, 0x023d, 0x023d, 0x023d, + 0x07d4, 0x07d4, 0x07d4, 0x07d4, 0x07d4, 0x07d4, 0x07d4, 0x07d4, + 0xe93d, 0xe93d, 0xe93d, 0xe93d, 0xe93d, 0xe93d, 0xe93d, 0xe93d, + 0x43d4, 0x43d4, 0x43d4, 0x43d4, 0x43d4, 0x43d4, 0x43d4, 0x43d4, + 0x0108, 0x0108, 0x0108, 0x0108, 0x0108, 0x0108, 0x0108, 0x0108, + 0x017f, 0x017f, 0x017f, 0x017f, 0x017f, 0x017f, 0x017f, 0x017f, + 0x9908, 0x9908, 0x9908, 0x9908, 0x9908, 0x9908, 0x9908, 0x9908, + 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, + 0x04c7, 0x04c7, 0x04c7, 0x04c7, 0x028c, 0x028c, 0x028c, 0x028c, + 0x0ad9, 0x0ad9, 0x0ad9, 0x0ad9, 0x03f7, 0x03f7, 0x03f7, 0x03f7, + 0xe9c7, 0xe9c7, 0xe9c7, 0xe9c7, 0xe68c, 0xe68c, 0xe68c, 0xe68c, + 0x05d9, 0x05d9, 0x05d9, 0x05d9, 0x78f7, 0x78f7, 0x78f7, 0x78f7, + 0x07f4, 0x07f4, 0x07f4, 0x07f4, 0x05d3, 0x05d3, 0x05d3, 0x05d3, + 0x0be7, 0x0be7, 0x0be7, 0x0be7, 0x06f9, 0x06f9, 0x06f9, 0x06f9, + 0xa3f4, 0xa3f4, 0xa3f4, 0xa3f4, 0x4ed3, 0x4ed3, 0x4ed3, 0x4ed3, + 0x50e7, 0x50e7, 0x50e7, 0x50e7, 0x61f9, 0x61f9, 0x61f9, 0x61f9, + 0x09c4, 0x09c4, 0x09c4, 0x09c4, 0x09c4, 0x09c4, 0x09c4, 0x09c4, + 0x05b2, 0x05b2, 0x05b2, 0x05b2, 0x05b2, 0x05b2, 0x05b2, 0x05b2, + 0x15c4, 0x15c4, 0x15c4, 0x15c4, 0x15c4, 0x15c4, 0x15c4, 0x15c4, + 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, + 0x06bf, 0x06bf, 0x06bf, 0x06bf, 0x06bf, 0x06bf, 0x06bf, 0x06bf, + 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, + 0x53bf, 0x53bf, 0x53bf, 0x53bf, 0x53bf, 0x53bf, 0x53bf, 0x53bf, + 0x997f, 0x997f, 0x997f, 0x997f, 0x997f, 0x997f, 0x997f, 0x997f, + 0x0204, 0x0204, 0x0204, 0x0204, 0x0cf9, 0x0cf9, 0x0cf9, 0x0cf9, + 0x0bc1, 0x0bc1, 0x0bc1, 0x0bc1, 0x0a67, 0x0a67, 0x0a67, 0x0a67, + 0xce04, 0xce04, 0xce04, 0xce04, 0x67f9, 0x67f9, 0x67f9, 0x67f9, + 0x3ec1, 0x3ec1, 0x3ec1, 0x3ec1, 0xcf67, 0xcf67, 0xcf67, 0xcf67, + 0x06af, 0x06af, 0x06af, 0x06af, 0x0877, 0x0877, 0x0877, 0x0877, + 0x007e, 0x007e, 0x007e, 0x007e, 0x05bd, 0x05bd, 0x05bd, 0x05bd, + 0x23af, 0x23af, 0x23af, 0x23af, 0xfd77, 0xfd77, 0xfd77, 0xfd77, + 0x9a7e, 0x9a7e, 0x9a7e, 0x9a7e, 0x6cbd, 0x6cbd, 0x6cbd, 0x6cbd, + 0x08b2, 0x08b2, 0x01ae, 0x01ae, 0x022b, 0x022b, 0x034b, 0x034b, + 0x081e, 0x081e, 0x0367, 0x0367, 0x060e, 0x060e, 0x0069, 0x0069, + 0xfeb2, 0xfeb2, 0x2bae, 0x2bae, 0xd32b, 0xd32b, 0x344b, 0x344b, + 0x821e, 0x821e, 0xc867, 0xc867, 0x500e, 0x500e, 0xab69, 0xab69, + 0x01a6, 0x01a6, 0x024b, 0x024b, 0x00b1, 0x00b1, 0x0c16, 0x0c16, + 0x0bde, 0x0bde, 0x0b35, 0x0b35, 0x0626, 0x0626, 0x0675, 0x0675, + 0x93a6, 0x93a6, 0x334b, 0x334b, 0x03b1, 0x03b1, 0xee16, 0xee16, + 0xc5de, 0xc5de, 0x5a35, 0x5a35, 0x1826, 0x1826, 0x1575, 0x1575, + 0x0c0b, 0x0c0b, 0x030a, 0x030a, 0x0487, 0x0487, 0x0c6e, 0x0c6e, + 0x09f8, 0x09f8, 0x05cb, 0x05cb, 0x0aa7, 0x0aa7, 0x045f, 0x045f, + 0x7d0b, 0x7d0b, 0x810a, 0x810a, 0x2987, 0x2987, 0x766e, 0x766e, + 0x71f8, 0x71f8, 0xb6cb, 0xb6cb, 0x8fa7, 0x8fa7, 0x315f, 0x315f, + 0x06cb, 0x06cb, 0x0284, 0x0284, 0x0999, 0x0999, 0x015d, 0x015d, + 0x01a2, 0x01a2, 0x0149, 0x0149, 0x0c65, 0x0c65, 0x0cb6, 0x0cb6, + 0xb7cb, 0xb7cb, 0x4e84, 0x4e84, 0x4499, 0x4499, 0x485d, 0x485d, + 0xc7a2, 0xc7a2, 0x4c49, 0x4c49, 0xeb65, 0xeb65, 0xceb6, 0xceb6, + 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, + 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, + 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, + 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, + 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, + 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, + 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, + 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, + 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, + 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, + 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, + 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, + 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, + 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, + 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, + 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, + 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, + 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, + 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, + 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, + 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, + 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, + 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, + 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, + 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, + 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, + 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, + 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, + 0x0a58, 0x0a58, 0x0a58, 0x0a58, 0x0a58, 0x0a58, 0x0a58, 0x0a58, + 0x03f9, 0x03f9, 0x03f9, 0x03f9, 0x03f9, 0x03f9, 0x03f9, 0x03f9, + 0x9258, 0x9258, 0x9258, 0x9258, 0x9258, 0x9258, 0x9258, 0x9258, + 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, + 0x02dc, 0x02dc, 0x02dc, 0x02dc, 0x02dc, 0x02dc, 0x02dc, 0x02dc, + 0x0260, 0x0260, 0x0260, 0x0260, 0x0260, 0x0260, 0x0260, 0x0260, + 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, + 0x2260, 0x2260, 0x2260, 0x2260, 0x2260, 0x2260, 0x2260, 0x2260, + 0x09ac, 0x09ac, 0x09ac, 0x09ac, 0x0ca7, 0x0ca7, 0x0ca7, 0x0ca7, + 0x0bf2, 0x0bf2, 0x0bf2, 0x0bf2, 0x033e, 0x033e, 0x033e, 0x033e, + 0x4dac, 0x4dac, 0x4dac, 0x4dac, 0x91a7, 0x91a7, 0x91a7, 0x91a7, + 0xc1f2, 0xc1f2, 0xc1f2, 0xc1f2, 0xdd3e, 0xdd3e, 0xdd3e, 0xdd3e, + 0x006b, 0x006b, 0x006b, 0x006b, 0x0774, 0x0774, 0x0774, 0x0774, + 0x0c0a, 0x0c0a, 0x0c0a, 0x0c0a, 0x094a, 0x094a, 0x094a, 0x094a, + 0x916b, 0x916b, 0x916b, 0x916b, 0x2374, 0x2374, 0x2374, 0x2374, + 0x8a0a, 0x8a0a, 0x8a0a, 0x8a0a, 0x474a, 0x474a, 0x474a, 0x474a, + 0x06fb, 0x06fb, 0x06fb, 0x06fb, 0x06fb, 0x06fb, 0x06fb, 0x06fb, + 0x019b, 0x019b, 0x019b, 0x019b, 0x019b, 0x019b, 0x019b, 0x019b, + 0x47fb, 0x47fb, 0x47fb, 0x47fb, 0x47fb, 0x47fb, 0x47fb, 0x47fb, + 0x229b, 0x229b, 0x229b, 0x229b, 0x229b, 0x229b, 0x229b, 0x229b, + 0x0c34, 0x0c34, 0x0c34, 0x0c34, 0x0c34, 0x0c34, 0x0c34, 0x0c34, + 0x06de, 0x06de, 0x06de, 0x06de, 0x06de, 0x06de, 0x06de, 0x06de, + 0x6834, 0x6834, 0x6834, 0x6834, 0x6834, 0x6834, 0x6834, 0x6834, + 0xc0de, 0xc0de, 0xc0de, 0xc0de, 0xc0de, 0xc0de, 0xc0de, 0xc0de, + 0x0b73, 0x0b73, 0x0b73, 0x0b73, 0x03c1, 0x03c1, 0x03c1, 0x03c1, + 0x071d, 0x071d, 0x071d, 0x071d, 0x0a2c, 0x0a2c, 0x0a2c, 0x0a2c, + 0x3473, 0x3473, 0x3473, 0x3473, 0x36c1, 0x36c1, 0x36c1, 0x36c1, + 0x8e1d, 0x8e1d, 0x8e1d, 0x8e1d, 0xce2c, 0xce2c, 0xce2c, 0xce2c, + 0x01c0, 0x01c0, 0x01c0, 0x01c0, 0x08d8, 0x08d8, 0x08d8, 0x08d8, + 0x02a5, 0x02a5, 0x02a5, 0x02a5, 0x0806, 0x0806, 0x0806, 0x0806, + 0x41c0, 0x41c0, 0x41c0, 0x41c0, 0x10d8, 0x10d8, 0x10d8, 0x10d8, + 0xa1a5, 0xa1a5, 0xa1a5, 0xa1a5, 0xba06, 0xba06, 0xba06, 0xba06, + 0x0331, 0x0331, 0x0449, 0x0449, 0x025b, 0x025b, 0x0262, 0x0262, + 0x052a, 0x052a, 0x07fc, 0x07fc, 0x0748, 0x0748, 0x0180, 0x0180, + 0x8631, 0x8631, 0x4f49, 0x4f49, 0x635b, 0x635b, 0x0862, 0x0862, + 0xe32a, 0xe32a, 0x3bfc, 0x3bfc, 0x5f48, 0x5f48, 0x8180, 0x8180, + 0x0842, 0x0842, 0x0c79, 0x0c79, 0x04c2, 0x04c2, 0x07ca, 0x07ca, + 0x0997, 0x0997, 0x00dc, 0x00dc, 0x085e, 0x085e, 0x0686, 0x0686, + 0xae42, 0xae42, 0xe779, 0xe779, 0x2ac2, 0x2ac2, 0xc5ca, 0xc5ca, + 0x5e97, 0x5e97, 0xd4dc, 0xd4dc, 0x425e, 0x425e, 0x3886, 0x3886, + 0x0860, 0x0860, 0x0707, 0x0707, 0x0803, 0x0803, 0x031a, 0x031a, + 0x071b, 0x071b, 0x09ab, 0x09ab, 0x099b, 0x099b, 0x01de, 0x01de, + 0x2860, 0x2860, 0xac07, 0xac07, 0xe103, 0xe103, 0xb11a, 0xb11a, + 0xa81b, 0xa81b, 0x5aab, 0x5aab, 0x2a9b, 0x2a9b, 0xbbde, 0xbbde, + 0x0c95, 0x0c95, 0x0bcd, 0x0bcd, 0x03e4, 0x03e4, 0x03df, 0x03df, + 0x03be, 0x03be, 0x074d, 0x074d, 0x05f2, 0x05f2, 0x065c, 0x065c, + 0x7b95, 0x7b95, 0xa2cd, 0xa2cd, 0x6fe4, 0x6fe4, 0xb0df, 0xb0df, + 0x5dbe, 0x5dbe, 0x1e4d, 0x1e4d, 0xbbf2, 0xbbf2, 0x5a5c, 0x5a5c, +}; + +XALIGNED(16) static const word16 L_mlkem_avx2_zetas_basemul[] WC_X64I_UNUSED = { + 0x08b2, 0x081e, 0xf74e, 0xf7e2, 0x01ae, 0x0367, 0xfe52, 0xfc99, + 0x022b, 0x060e, 0xfdd5, 0xf9f2, 0x034b, 0x0069, 0xfcb5, 0xff97, + 0xfeb2, 0x821e, 0x014e, 0x7de2, 0x2bae, 0xc867, 0xd452, 0x3799, + 0xd32b, 0x500e, 0x2cd5, 0xaff2, 0x344b, 0xab69, 0xcbb5, 0x5497, + 0x01a6, 0x0bde, 0xfe5a, 0xf422, 0x024b, 0x0b35, 0xfdb5, 0xf4cb, + 0x00b1, 0x0626, 0xff4f, 0xf9da, 0x0c16, 0x0675, 0xf3ea, 0xf98b, + 0x93a6, 0xc5de, 0x6c5a, 0x3a22, 0x334b, 0x5a35, 0xccb5, 0xa5cb, + 0x03b1, 0x1826, 0xfc4f, 0xe7da, 0xee16, 0x1575, 0x11ea, 0xea8b, + 0x0c0b, 0x09f8, 0xf3f5, 0xf608, 0x030a, 0x05cb, 0xfcf6, 0xfa35, + 0x0487, 0x0aa7, 0xfb79, 0xf559, 0x0c6e, 0x045f, 0xf392, 0xfba1, + 0x7d0b, 0x71f8, 0x82f5, 0x8e08, 0x810a, 0xb6cb, 0x7ef6, 0x4935, + 0x2987, 0x8fa7, 0xd679, 0x7059, 0x766e, 0x315f, 0x8992, 0xcea1, + 0x06cb, 0x01a2, 0xf935, 0xfe5e, 0x0284, 0x0149, 0xfd7c, 0xfeb7, + 0x0999, 0x0c65, 0xf667, 0xf39b, 0x015d, 0x0cb6, 0xfea3, 0xf34a, + 0xb7cb, 0xc7a2, 0x4835, 0x385e, 0x4e84, 0x4c49, 0xb17c, 0xb3b7, + 0x4499, 0xeb65, 0xbb67, 0x149b, 0x485d, 0xceb6, 0xb7a3, 0x314a, + 0x0331, 0x052a, 0xfccf, 0xfad6, 0x0449, 0x07fc, 0xfbb7, 0xf804, + 0x025b, 0x0748, 0xfda5, 0xf8b8, 0x0262, 0x0180, 0xfd9e, 0xfe80, + 0x8631, 0xe32a, 0x79cf, 0x1cd6, 0x4f49, 0x3bfc, 0xb0b7, 0xc404, + 0x635b, 0x5f48, 0x9ca5, 0xa0b8, 0x0862, 0x8180, 0xf79e, 0x7e80, + 0x0842, 0x0997, 0xf7be, 0xf669, 0x0c79, 0x00dc, 0xf387, 0xff24, + 0x04c2, 0x085e, 0xfb3e, 0xf7a2, 0x07ca, 0x0686, 0xf836, 0xf97a, + 0xae42, 0x5e97, 0x51be, 0xa169, 0xe779, 0xd4dc, 0x1887, 0x2b24, + 0x2ac2, 0x425e, 0xd53e, 0xbda2, 0xc5ca, 0x3886, 0x3a36, 0xc77a, + 0x0860, 0x071b, 0xf7a0, 0xf8e5, 0x0707, 0x09ab, 0xf8f9, 0xf655, + 0x0803, 0x099b, 0xf7fd, 0xf665, 0x031a, 0x01de, 0xfce6, 0xfe22, + 0x2860, 0xa81b, 0xd7a0, 0x57e5, 0xac07, 0x5aab, 0x53f9, 0xa555, + 0xe103, 0x2a9b, 0x1efd, 0xd565, 0xb11a, 0xbbde, 0x4ee6, 0x4422, + 0x0c95, 0x03be, 0xf36b, 0xfc42, 0x0bcd, 0x074d, 0xf433, 0xf8b3, + 0x03e4, 0x05f2, 0xfc1c, 0xfa0e, 0x03df, 0x065c, 0xfc21, 0xf9a4, + 0x7b95, 0x5dbe, 0x846b, 0xa242, 0xa2cd, 0x1e4d, 0x5d33, 0xe1b3, + 0x6fe4, 0xbbf2, 0x901c, 0x440e, 0xb0df, 0x5a5c, 0x4f21, 0xa5a4, +}; + +XALIGNED(16) static const word16 L_mlkem_avx2_zetas_inv[] WC_X64I_UNUSED = { + 0x06a5, 0x06a5, 0x05b4, 0x05b4, 0x070f, 0x070f, 0x0943, 0x0943, + 0x0922, 0x0922, 0x0134, 0x0134, 0x091d, 0x091d, 0x006c, 0x006c, + 0xa5a5, 0xa5a5, 0xe1b4, 0xe1b4, 0x440f, 0x440f, 0xa243, 0xa243, + 0x4f22, 0x4f22, 0x5d34, 0x5d34, 0x901d, 0x901d, 0x846c, 0x846c, + 0x0b23, 0x0b23, 0x0356, 0x0356, 0x0366, 0x0366, 0x05e6, 0x05e6, + 0x09e7, 0x09e7, 0x05fa, 0x05fa, 0x04fe, 0x04fe, 0x04a1, 0x04a1, + 0x4423, 0x4423, 0xa556, 0xa556, 0xd566, 0xd566, 0x57e6, 0x57e6, + 0x4ee7, 0x4ee7, 0x53fa, 0x53fa, 0x1efe, 0x1efe, 0xd7a1, 0xd7a1, + 0x04fb, 0x04fb, 0x04fb, 0x04fb, 0x0a5c, 0x0a5c, 0x0a5c, 0x0a5c, + 0x0429, 0x0429, 0x0429, 0x0429, 0x0b41, 0x0b41, 0x0b41, 0x0b41, + 0x45fb, 0x45fb, 0x45fb, 0x45fb, 0x5e5c, 0x5e5c, 0x5e5c, 0x5e5c, + 0xef29, 0xef29, 0xef29, 0xef29, 0xbe41, 0xbe41, 0xbe41, 0xbe41, + 0x02d5, 0x02d5, 0x02d5, 0x02d5, 0x05e4, 0x05e4, 0x05e4, 0x05e4, + 0x0940, 0x0940, 0x0940, 0x0940, 0x018e, 0x018e, 0x018e, 0x018e, + 0x31d5, 0x31d5, 0x31d5, 0x31d5, 0x71e4, 0x71e4, 0x71e4, 0x71e4, + 0xc940, 0xc940, 0xc940, 0xc940, 0xcb8e, 0xcb8e, 0xcb8e, 0xcb8e, + 0x0623, 0x0623, 0x0623, 0x0623, 0x0623, 0x0623, 0x0623, 0x0623, + 0x00cd, 0x00cd, 0x00cd, 0x00cd, 0x00cd, 0x00cd, 0x00cd, 0x00cd, + 0x3f23, 0x3f23, 0x3f23, 0x3f23, 0x3f23, 0x3f23, 0x3f23, 0x3f23, + 0x97cd, 0x97cd, 0x97cd, 0x97cd, 0x97cd, 0x97cd, 0x97cd, 0x97cd, + 0x0b66, 0x0b66, 0x0b66, 0x0b66, 0x0b66, 0x0b66, 0x0b66, 0x0b66, + 0x0606, 0x0606, 0x0606, 0x0606, 0x0606, 0x0606, 0x0606, 0x0606, + 0xdd66, 0xdd66, 0xdd66, 0xdd66, 0xdd66, 0xdd66, 0xdd66, 0xdd66, + 0xb806, 0xb806, 0xb806, 0xb806, 0xb806, 0xb806, 0xb806, 0xb806, + 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, + 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, + 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, + 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, + 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, + 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, + 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, + 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, + 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, + 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, + 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, + 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, + 0x067b, 0x067b, 0x0c25, 0x0c25, 0x04a3, 0x04a3, 0x036a, 0x036a, + 0x0537, 0x0537, 0x0088, 0x0088, 0x083f, 0x083f, 0x04bf, 0x04bf, + 0xc77b, 0xc77b, 0x2b25, 0x2b25, 0xbda3, 0xbda3, 0xa16a, 0xa16a, + 0x3a37, 0x3a37, 0x1888, 0x1888, 0xd53f, 0xd53f, 0x51bf, 0x51bf, + 0x0b81, 0x0b81, 0x0505, 0x0505, 0x05b9, 0x05b9, 0x07d7, 0x07d7, + 0x0a9f, 0x0a9f, 0x08b8, 0x08b8, 0x0aa6, 0x0aa6, 0x09d0, 0x09d0, + 0x7e81, 0x7e81, 0xc405, 0xc405, 0xa0b9, 0xa0b9, 0x1cd7, 0x1cd7, + 0xf79f, 0xf79f, 0xb0b8, 0xb0b8, 0x9ca6, 0x9ca6, 0x79d0, 0x79d0, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x00f7, 0x00f7, 0x00f7, 0x00f7, + 0x058d, 0x058d, 0x058d, 0x058d, 0x0c96, 0x0c96, 0x0c96, 0x0c96, + 0xb8b7, 0xb8b7, 0xb8b7, 0xb8b7, 0x75f7, 0x75f7, 0x75f7, 0x75f7, + 0xdc8d, 0xdc8d, 0xdc8d, 0xdc8d, 0x6e96, 0x6e96, 0x6e96, 0x6e96, + 0x09c3, 0x09c3, 0x09c3, 0x09c3, 0x010f, 0x010f, 0x010f, 0x010f, + 0x005a, 0x005a, 0x005a, 0x005a, 0x0355, 0x0355, 0x0355, 0x0355, + 0x22c3, 0x22c3, 0x22c3, 0x22c3, 0x3e0f, 0x3e0f, 0x3e0f, 0x3e0f, + 0x6e5a, 0x6e5a, 0x6e5a, 0x6e5a, 0xb255, 0xb255, 0xb255, 0xb255, + 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, + 0x0a25, 0x0a25, 0x0a25, 0x0a25, 0x0a25, 0x0a25, 0x0a25, 0x0a25, + 0xdda1, 0xdda1, 0xdda1, 0xdda1, 0xdda1, 0xdda1, 0xdda1, 0xdda1, + 0x2925, 0x2925, 0x2925, 0x2925, 0x2925, 0x2925, 0x2925, 0x2925, + 0x0908, 0x0908, 0x0908, 0x0908, 0x0908, 0x0908, 0x0908, 0x0908, + 0x02a9, 0x02a9, 0x02a9, 0x02a9, 0x02a9, 0x02a9, 0x02a9, 0x02a9, + 0xa108, 0xa108, 0xa108, 0xa108, 0xa108, 0xa108, 0xa108, 0xa108, + 0x6da9, 0x6da9, 0x6da9, 0x6da9, 0x6da9, 0x6da9, 0x6da9, 0x6da9, + 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, + 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, + 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, + 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, + 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, + 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, + 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, + 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, + 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, + 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, + 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, + 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, + 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, + 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, + 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, + 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, + 0x004b, 0x004b, 0x0bb8, 0x0bb8, 0x009c, 0x009c, 0x0b5f, 0x0b5f, + 0x0ba4, 0x0ba4, 0x0a7d, 0x0a7d, 0x0368, 0x0368, 0x0636, 0x0636, + 0x314b, 0x314b, 0xb3b8, 0xb3b8, 0x149c, 0x149c, 0x385f, 0x385f, + 0xb7a4, 0xb7a4, 0xb17d, 0xb17d, 0xbb68, 0xbb68, 0x4836, 0x4836, + 0x08a2, 0x08a2, 0x0736, 0x0736, 0x025a, 0x025a, 0x0309, 0x0309, + 0x0093, 0x0093, 0x09f7, 0x09f7, 0x087a, 0x087a, 0x00f6, 0x00f6, + 0xcea2, 0xcea2, 0x4936, 0x4936, 0x705a, 0x705a, 0x8e09, 0x8e09, + 0x8993, 0x8993, 0x7ef7, 0x7ef7, 0xd67a, 0xd67a, 0x82f6, 0x82f6, + 0x0744, 0x0744, 0x0744, 0x0744, 0x0c83, 0x0c83, 0x0c83, 0x0c83, + 0x048a, 0x048a, 0x048a, 0x048a, 0x0652, 0x0652, 0x0652, 0x0652, + 0x9344, 0x9344, 0x9344, 0x9344, 0x6583, 0x6583, 0x6583, 0x6583, + 0x028a, 0x028a, 0x028a, 0x028a, 0xdc52, 0xdc52, 0xdc52, 0xdc52, + 0x029a, 0x029a, 0x029a, 0x029a, 0x0140, 0x0140, 0x0140, 0x0140, + 0x0008, 0x0008, 0x0008, 0x0008, 0x0afd, 0x0afd, 0x0afd, 0x0afd, + 0x309a, 0x309a, 0x309a, 0x309a, 0xc140, 0xc140, 0xc140, 0xc140, + 0x9808, 0x9808, 0x9808, 0x9808, 0x31fd, 0x31fd, 0x31fd, 0x31fd, + 0x0082, 0x0082, 0x0082, 0x0082, 0x0082, 0x0082, 0x0082, 0x0082, + 0x0642, 0x0642, 0x0642, 0x0642, 0x0642, 0x0642, 0x0642, 0x0642, + 0x6682, 0x6682, 0x6682, 0x6682, 0x6682, 0x6682, 0x6682, 0x6682, + 0xac42, 0xac42, 0xac42, 0xac42, 0xac42, 0xac42, 0xac42, 0xac42, + 0x074f, 0x074f, 0x074f, 0x074f, 0x074f, 0x074f, 0x074f, 0x074f, + 0x033d, 0x033d, 0x033d, 0x033d, 0x033d, 0x033d, 0x033d, 0x033d, + 0x044f, 0x044f, 0x044f, 0x044f, 0x044f, 0x044f, 0x044f, 0x044f, + 0xea3d, 0xea3d, 0xea3d, 0xea3d, 0xea3d, 0xea3d, 0xea3d, 0xea3d, + 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, + 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, + 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, + 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, + 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, + 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, + 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, + 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, + 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, + 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, + 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, + 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, + 0x068c, 0x068c, 0x01cc, 0x01cc, 0x06db, 0x06db, 0x0123, 0x0123, + 0x00eb, 0x00eb, 0x0ab6, 0x0ab6, 0x0c50, 0x0c50, 0x0b5b, 0x0b5b, + 0xea8c, 0xea8c, 0xa5cc, 0xa5cc, 0xe7db, 0xe7db, 0x3a23, 0x3a23, + 0x11eb, 0x11eb, 0xccb6, 0xccb6, 0xfc50, 0xfc50, 0x6c5b, 0x6c5b, + 0x0c98, 0x0c98, 0x099a, 0x099a, 0x06f3, 0x06f3, 0x04e3, 0x04e3, + 0x09b6, 0x09b6, 0x0b53, 0x0b53, 0x0ad6, 0x0ad6, 0x044f, 0x044f, + 0x5498, 0x5498, 0x379a, 0x379a, 0xaff3, 0xaff3, 0x7de3, 0x7de3, + 0xcbb6, 0xcbb6, 0xd453, 0xd453, 0x2cd6, 0x2cd6, 0x014f, 0x014f, + 0x0608, 0x0608, 0x0608, 0x0608, 0x011a, 0x011a, 0x011a, 0x011a, + 0x072e, 0x072e, 0x072e, 0x072e, 0x050d, 0x050d, 0x050d, 0x050d, + 0x9e08, 0x9e08, 0x9e08, 0x9e08, 0xaf1a, 0xaf1a, 0xaf1a, 0xaf1a, + 0xb12e, 0xb12e, 0xb12e, 0xb12e, 0x5c0d, 0x5c0d, 0x5c0d, 0x5c0d, + 0x090a, 0x090a, 0x090a, 0x090a, 0x0228, 0x0228, 0x0228, 0x0228, + 0x0a75, 0x0a75, 0x0a75, 0x0a75, 0x083a, 0x083a, 0x083a, 0x083a, + 0x870a, 0x870a, 0x870a, 0x870a, 0xfa28, 0xfa28, 0xfa28, 0xfa28, + 0x1975, 0x1975, 0x1975, 0x1975, 0x163a, 0x163a, 0x163a, 0x163a, + 0x0b82, 0x0b82, 0x0b82, 0x0b82, 0x0b82, 0x0b82, 0x0b82, 0x0b82, + 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, + 0x7182, 0x7182, 0x7182, 0x7182, 0x7182, 0x7182, 0x7182, 0x7182, + 0x66f9, 0x66f9, 0x66f9, 0x66f9, 0x66f9, 0x66f9, 0x66f9, 0x66f9, + 0x052d, 0x052d, 0x052d, 0x052d, 0x052d, 0x052d, 0x052d, 0x052d, + 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, + 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, + 0x16c4, 0x16c4, 0x16c4, 0x16c4, 0x16c4, 0x16c4, 0x16c4, 0x16c4, + 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, + 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, + 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, + 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, + 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, + 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, + 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, + 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, + 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, + 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, + 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, + 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, + 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, + 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, + 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, + 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, + 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, + 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, + 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, + 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, + 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, + 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, + 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, + 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_keygen_avx2(sword16* priv, sword16* pub, sword16* e, + const sword16* a, int kp) +{ + word64 rdi, rsi, rdx, rcx, r8, r11, r9, r10, r12 = 0, rax = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), y14, + y15; + + rdi = (word64)(size_t)priv; + rsi = (word64)(size_t)pub; + rdx = (word64)(size_t)e; + rcx = (word64)(size_t)a; + r8 = (word64)(word32)kp; + + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_q, 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_v, 0)); + r11 = (word64)(rdi); + r9 = (word64)((word64)(sword64)(sword32)(word32)r8); + r10 = (word64)(rdi); +L_mlkem_keygen_avx2_priv: + /* ntt */ + r12 = (word64)((word64)(size_t)L_mlkem_avx2_zetas); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), y7); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + /* 64: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 64)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 96)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 160)); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 192)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 224)); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + /* 16: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 352)); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 16: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 384)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 416)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 448)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 480)); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 8: 1/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 512)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 544)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 576)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 608)); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y0 = _mm256_sub_epi16(y1, y0); + y2 = _mm256_sub_epi16(y3, y2); + y1 = _mm256_sub_epi16(y8, y0); + y3 = _mm256_sub_epi16(y9, y2); + y8 = _mm256_add_epi16(y8, y0); + y9 = _mm256_add_epi16(y9, y2); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 640)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 672)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 704)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 736)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 8: 1/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 768)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 800)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 832)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 864)); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y4 = _mm256_sub_epi16(y5, y4); + y6 = _mm256_sub_epi16(y7, y6); + y5 = _mm256_sub_epi16(y8, y4); + y7 = _mm256_sub_epi16(y9, y6); + y8 = _mm256_add_epi16(y8, y4); + y9 = _mm256_add_epi16(y9, y6); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 896)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 928)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 960)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 992)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 2: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1024)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1056)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1088)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1120)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 2: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1184)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1216)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1248)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_mulhi_epi16(y0, y15); + y9 = _mm256_mulhi_epi16(y1, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y0, y8); + y9 = _mm256_sub_epi16(y1, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), y9); + y8 = _mm256_mulhi_epi16(y2, y15); + y9 = _mm256_mulhi_epi16(y3, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y9); + y8 = _mm256_mulhi_epi16(y4, y15); + y9 = _mm256_mulhi_epi16(y5, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), y9); + y8 = _mm256_mulhi_epi16(y6, y15); + y9 = _mm256_mulhi_epi16(y7, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + /* 64: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1280)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1312)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1344)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1376)); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1408)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1440)); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + /* 16: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1568)); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 16: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1600)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1632)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1664)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1696)); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 8: 2/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1728)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1760)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1792)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1824)); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y0 = _mm256_sub_epi16(y1, y0); + y2 = _mm256_sub_epi16(y3, y2); + y1 = _mm256_sub_epi16(y8, y0); + y3 = _mm256_sub_epi16(y9, y2); + y8 = _mm256_add_epi16(y8, y0); + y9 = _mm256_add_epi16(y9, y2); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1856)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1888)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1920)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1952)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 8: 2/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1984)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2016)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2048)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2080)); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y4 = _mm256_sub_epi16(y5, y4); + y6 = _mm256_sub_epi16(y7, y6); + y5 = _mm256_sub_epi16(y8, y4); + y7 = _mm256_sub_epi16(y9, y6); + y8 = _mm256_add_epi16(y8, y4); + y9 = _mm256_add_epi16(y9, y6); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2112)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2144)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2176)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2208)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 2: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2240)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2272)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2304)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2336)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 2: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2400)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2432)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2464)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_mulhi_epi16(y0, y15); + y9 = _mm256_mulhi_epi16(y1, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y0, y8); + y9 = _mm256_sub_epi16(y1, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), y9); + y8 = _mm256_mulhi_epi16(y2, y15); + y9 = _mm256_mulhi_epi16(y3, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), y9); + y8 = _mm256_mulhi_epi16(y4, y15); + y9 = _mm256_mulhi_epi16(y5, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), y9); + y8 = _mm256_mulhi_epi16(y6, y15); + y9 = _mm256_mulhi_epi16(y7, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), y9); + r10 = (word64)(r10 + 0x200); + r9 = (word64)(r9 - 1); + if ((sword64)(r9) > (sword64)(0)) { + goto L_mlkem_keygen_avx2_priv; + } + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_qinv, 0)); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + r10 = (word64)(rsi); +L_mlkem_keygen_avx2_acc: + /* Pointwise acc mont */ + r9 = (word64)((word64)(sword64)(sword32)(word32)r8); + /* Base mul mont */ + r12 = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), y1); + rcx = (word64)(rcx + 0x200); + rdi = (word64)(rdi + 0x200); + r9 = (word64)(r9 - 2); + if ((r9) == (0)) { + goto L_pointwise_acc_mont_end_keygen; + } +L_pointwise_acc_mont_start_keygen: + /* Base mul mont add */ + r12 = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), y1); + rcx = (word64)(rcx + 0x200); + rdi = (word64)(rdi + 0x200); + r9 = (word64)(r9 - 1); + if ((sword64)(r9) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_keygen; + } +L_pointwise_acc_mont_end_keygen: + /* Base mul mont add */ + r12 = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y13); + y9 = _mm256_mullo_epi16(y2, y13); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), y1); + rcx = (word64)(rcx + 0x200); + rdi = (word64)(r11); + r10 = (word64)(r10 + 0x200); + rax = (word64)(rax - 1); + if ((sword64)(rax) > (sword64)(0)) { + goto L_mlkem_keygen_avx2_acc; + } + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_f, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_f_qinv, 0)); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + r10 = (word64)(rsi); +L_mlkem_keygen_avx2_to_mont: + /* To Mont */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y4 = _mm256_mullo_epi16(y0, y13); + y5 = _mm256_mulhi_epi16(y0, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y0 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y1, y13); + y5 = _mm256_mulhi_epi16(y1, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y1 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y2, y13); + y5 = _mm256_mulhi_epi16(y2, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y2 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y3, y13); + y5 = _mm256_mulhi_epi16(y3, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y3 = _mm256_sub_epi16(y5, y4); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y4 = _mm256_mullo_epi16(y0, y13); + y5 = _mm256_mulhi_epi16(y0, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y0 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y1, y13); + y5 = _mm256_mulhi_epi16(y1, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y1 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y2, y13); + y5 = _mm256_mulhi_epi16(y2, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y2 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y3, y13); + y5 = _mm256_mulhi_epi16(y3, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y3 = _mm256_sub_epi16(y5, y4); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y4 = _mm256_mullo_epi16(y0, y13); + y5 = _mm256_mulhi_epi16(y0, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y0 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y1, y13); + y5 = _mm256_mulhi_epi16(y1, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y1 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y2, y13); + y5 = _mm256_mulhi_epi16(y2, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y2 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y3, y13); + y5 = _mm256_mulhi_epi16(y3, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y3 = _mm256_sub_epi16(y5, y4); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y4 = _mm256_mullo_epi16(y0, y13); + y5 = _mm256_mulhi_epi16(y0, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y0 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y1, y13); + y5 = _mm256_mulhi_epi16(y1, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y1 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y2, y13); + y5 = _mm256_mulhi_epi16(y2, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y2 = _mm256_sub_epi16(y5, y4); + y4 = _mm256_mullo_epi16(y3, y13); + y5 = _mm256_mulhi_epi16(y3, y12); + y4 = _mm256_mulhi_epi16(y4, y14); + y3 = _mm256_sub_epi16(y5, y4); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), y3); + r10 = (word64)(r10 + 0x200); + rax = (word64)(rax - 1); + if ((sword64)(rax) > (sword64)(0)) { + goto L_mlkem_keygen_avx2_to_mont; + } + rax = (word64)((word64)(sword64)(sword32)(word32)r8); +L_mlkem_keygen_avx2_to_mont_ntt_err: + /* ntt */ + r12 = (word64)((word64)(size_t)L_mlkem_avx2_zetas); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y7); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + /* 64: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 64)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 96)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 160)); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 192)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 224)); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + /* 16: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 352)); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 16: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 384)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 416)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 448)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 480)); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 8: 1/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 512)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 544)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 576)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 608)); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y0 = _mm256_sub_epi16(y1, y0); + y2 = _mm256_sub_epi16(y3, y2); + y1 = _mm256_sub_epi16(y8, y0); + y3 = _mm256_sub_epi16(y9, y2); + y8 = _mm256_add_epi16(y8, y0); + y9 = _mm256_add_epi16(y9, y2); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 640)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 672)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 704)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 736)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 8: 1/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 768)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 800)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 832)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 864)); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y4 = _mm256_sub_epi16(y5, y4); + y6 = _mm256_sub_epi16(y7, y6); + y5 = _mm256_sub_epi16(y8, y4); + y7 = _mm256_sub_epi16(y9, y6); + y8 = _mm256_add_epi16(y8, y4); + y9 = _mm256_add_epi16(y9, y6); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 896)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 928)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 960)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 992)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 2: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1024)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1056)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1088)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1120)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 2: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1184)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1216)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1248)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mulhi_epi16(y0, y15); + y9 = _mm256_mulhi_epi16(y1, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y0, y8); + y9 = _mm256_sub_epi16(y1, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y9); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + y8 = _mm256_mulhi_epi16(y2, y15); + y9 = _mm256_mulhi_epi16(y3, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y9); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + y8 = _mm256_mulhi_epi16(y4, y15); + y9 = _mm256_mulhi_epi16(y5, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y9); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y6 = _mm256_add_epi16(y6, y8); + y7 = _mm256_add_epi16(y7, y9); + y8 = _mm256_mulhi_epi16(y6, y15); + y9 = _mm256_mulhi_epi16(y7, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + /* 64: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1280)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1312)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1344)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1376)); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1408)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1440)); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + /* 16: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1568)); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 16: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1600)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1632)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1664)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1696)); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 8: 2/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1728)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1760)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1792)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1824)); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y0 = _mm256_sub_epi16(y1, y0); + y2 = _mm256_sub_epi16(y3, y2); + y1 = _mm256_sub_epi16(y8, y0); + y3 = _mm256_sub_epi16(y9, y2); + y8 = _mm256_add_epi16(y8, y0); + y9 = _mm256_add_epi16(y9, y2); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1856)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1888)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1920)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1952)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 8: 2/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 1984)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2016)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2048)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2080)); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y4 = _mm256_sub_epi16(y5, y4); + y6 = _mm256_sub_epi16(y7, y6); + y5 = _mm256_sub_epi16(y8, y4); + y7 = _mm256_sub_epi16(y9, y6); + y8 = _mm256_add_epi16(y8, y4); + y9 = _mm256_add_epi16(y9, y6); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2112)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2144)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2176)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2208)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 2: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2240)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2272)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2304)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2336)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 2: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2400)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2432)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r12, 2464)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mulhi_epi16(y0, y15); + y9 = _mm256_mulhi_epi16(y1, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y0, y8); + y9 = _mm256_sub_epi16(y1, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y9); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + y8 = _mm256_mulhi_epi16(y2, y15); + y9 = _mm256_mulhi_epi16(y3, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y9); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + y8 = _mm256_mulhi_epi16(y4, y15); + y9 = _mm256_mulhi_epi16(y5, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y9); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y6 = _mm256_add_epi16(y6, y8); + y7 = _mm256_add_epi16(y7, y9); + y8 = _mm256_mulhi_epi16(y6, y15); + y9 = _mm256_mulhi_epi16(y7, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y9); + rdx = (word64)(rdx + 0x200); + rsi = (word64)(rsi + 0x200); + rax = (word64)(rax - 1); + if ((sword64)(rax) > (sword64)(0)) { + goto L_mlkem_keygen_avx2_to_mont_ntt_err; + } +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_encapsulate_avx2(const sword16* pub, sword16* bp, + sword16* vp, const sword16* at, sword16* sp, const sword16* ep, + const sword16* epp, const sword16* m, int kp) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11, rsp, r15, r13, r14, + rbx = 0, r12 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), y14, + y15; + XALIGNED(32) WC_X64I_SLOT stk[8]; + + rdi = (word64)(size_t)pub; + rsi = (word64)(size_t)bp; + rdx = (word64)(size_t)vp; + rcx = (word64)(size_t)at; + r8 = (word64)(size_t)sp; + r9 = (word64)(size_t)ep; + rax = (word64)(size_t)epp; + r10 = (word64)(size_t)m; + r11 = (word64)(word32)kp; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 48); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_q, 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_v, 0)); + r15 = (word64)(r8); + r13 = (word64)((word64)(sword64)(sword32)(word32)r11); + r14 = (word64)(r8); +L_mlkem_encapsulate_avx2_trans: + /* ntt */ + rbx = (word64)((word64)(size_t)L_mlkem_avx2_zetas); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 224)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 480)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 224), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 352)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 352), y7); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 224)); + /* 64: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + /* 16: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 352)); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 16: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 384)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 416)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 448)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 480)); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 8: 1/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 512)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 544)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 576)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 608)); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y0 = _mm256_sub_epi16(y1, y0); + y2 = _mm256_sub_epi16(y3, y2); + y1 = _mm256_sub_epi16(y8, y0); + y3 = _mm256_sub_epi16(y9, y2); + y8 = _mm256_add_epi16(y8, y0); + y9 = _mm256_add_epi16(y9, y2); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 640)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 672)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 704)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 736)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 8: 1/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 768)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 800)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 832)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 864)); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y4 = _mm256_sub_epi16(y5, y4); + y6 = _mm256_sub_epi16(y7, y6); + y5 = _mm256_sub_epi16(y8, y4); + y7 = _mm256_sub_epi16(y9, y6); + y8 = _mm256_add_epi16(y8, y4); + y9 = _mm256_add_epi16(y9, y6); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 896)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 928)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 960)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 992)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 2: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1024)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1056)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1088)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1120)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 2: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1184)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1216)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1248)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r14, 480)); + /* 64: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1280)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1312)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1344)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1376)); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1408)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1440)); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + /* 16: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1568)); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 16: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1600)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1632)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1664)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1696)); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 8: 2/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1728)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1760)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1792)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1824)); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y0 = _mm256_sub_epi16(y1, y0); + y2 = _mm256_sub_epi16(y3, y2); + y1 = _mm256_sub_epi16(y8, y0); + y3 = _mm256_sub_epi16(y9, y2); + y8 = _mm256_add_epi16(y8, y0); + y9 = _mm256_add_epi16(y9, y2); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1856)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1888)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1920)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1952)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 8: 2/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1984)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2016)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2048)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2080)); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y4 = _mm256_sub_epi16(y5, y4); + y6 = _mm256_sub_epi16(y7, y6); + y5 = _mm256_sub_epi16(y8, y4); + y7 = _mm256_sub_epi16(y9, y6); + y8 = _mm256_add_epi16(y8, y4); + y9 = _mm256_add_epi16(y9, y6); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2112)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2144)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2176)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2208)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 2: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2240)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2272)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2304)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2336)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 2: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2400)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2432)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2464)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 480), y7); + r14 = (word64)(r14 + 0x200); + r13 = (word64)(r13 - 1); + if ((sword64)(r13) > (sword64)(0)) { + goto L_mlkem_encapsulate_avx2_trans; + } + r12 = (word64)((word64)(sword64)(sword32)(word32)r11); +L_mlkem_encapsulate_avx2_calc: + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_qinv, 0)); + /* Pointwise acc mont */ + r13 = (word64)((word64)(sword64)(sword32)(word32)r11); + /* Base mul mont */ + rbx = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y1); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 2); + if ((r13) == (0)) { + goto L_pointwise_acc_mont_end_encap_bp; + } +L_pointwise_acc_mont_start_encap_bp: + /* Base mul mont add */ + rbx = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y1); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 1); + if ((sword64)(r13) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_encap_bp; + } +L_pointwise_acc_mont_end_encap_bp: + /* Base mul mont add */ + rbx = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y1); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(r15); + /* invntt */ + rbx = (word64)((word64)(size_t)L_mlkem_avx2_zetas_inv); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + /* 2: 1/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y9 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y0 = _mm256_slli_epi64(y9, 32); + y1 = _mm256_srli_epi64(y8, 32); + y0 = _mm256_blend_epi32(y8, y0, 0xaa); + y1 = _mm256_blend_epi32(y9, y1, 0x55); + y8 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y2 = _mm256_slli_epi64(y9, 32); + y3 = _mm256_srli_epi64(y8, 32); + y2 = _mm256_blend_epi32(y8, y2, 0xaa); + y3 = _mm256_blend_epi32(y9, y3, 0x55); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y0 = _mm256_unpacklo_epi32(y8, y1); + y1 = _mm256_unpackhi_epi32(y8, y1); + y2 = _mm256_unpacklo_epi32(y9, y3); + y3 = _mm256_unpackhi_epi32(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 8: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 352)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 16: 1/2 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 384)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 416)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 448)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 480)); + y8 = _mm256_sub_epi16(y0, y1); + y9 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_add_epi16(y0, y1); + y2 = _mm256_add_epi16(y2, y3); + y1 = _mm256_mullo_epi16(y8, y12); + y3 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y1 = _mm256_mulhi_epi16(y1, y14); + y3 = _mm256_mulhi_epi16(y3, y14); + y1 = _mm256_sub_epi16(y8, y1); + y3 = _mm256_sub_epi16(y9, y3); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 512)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 544)); + y8 = _mm256_add_epi16(y0, y2); + y9 = _mm256_add_epi16(y1, y3); + y2 = _mm256_sub_epi16(y0, y2); + y3 = _mm256_sub_epi16(y1, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y1 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y9, y1); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + /* 2: 1/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 576)); + y9 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 608)); + y4 = _mm256_slli_epi64(y9, 32); + y5 = _mm256_srli_epi64(y8, 32); + y4 = _mm256_blend_epi32(y8, y4, 0xaa); + y5 = _mm256_blend_epi32(y9, y5, 0x55); + y8 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 640)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 672)); + y6 = _mm256_slli_epi64(y9, 32); + y7 = _mm256_srli_epi64(y8, 32); + y6 = _mm256_blend_epi32(y8, y6, 0xaa); + y7 = _mm256_blend_epi32(y9, y7, 0x55); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 704)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 736)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 768)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 800)); + y4 = _mm256_unpacklo_epi32(y8, y5); + y5 = _mm256_unpackhi_epi32(y8, y5); + y6 = _mm256_unpacklo_epi32(y9, y7); + y7 = _mm256_unpackhi_epi32(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 8: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 832)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 864)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 896)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 928)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 16: 1/2 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 960)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 992)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1024)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1056)); + y8 = _mm256_sub_epi16(y4, y5); + y9 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_add_epi16(y4, y5); + y6 = _mm256_add_epi16(y6, y7); + y5 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y5 = _mm256_mulhi_epi16(y5, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y5 = _mm256_sub_epi16(y8, y5); + y7 = _mm256_sub_epi16(y9, y7); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1088)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1120)); + y8 = _mm256_add_epi16(y4, y6); + y9 = _mm256_add_epi16(y5, y7); + y6 = _mm256_sub_epi16(y4, y6); + y7 = _mm256_sub_epi16(y5, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y5 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y5 = _mm256_srai_epi16(y5, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y5 = _mm256_mullo_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + /* 64: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1184)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_sub_epi16(y2, y6); + y9 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_add_epi16(y2, y6); + y3 = _mm256_add_epi16(y3, y7); + y6 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y6 = _mm256_sub_epi16(y8, y6); + y7 = _mm256_sub_epi16(y9, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + /* 2: 2/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1216)); + y9 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1248)); + y0 = _mm256_slli_epi64(y9, 32); + y1 = _mm256_srli_epi64(y8, 32); + y0 = _mm256_blend_epi32(y8, y0, 0xaa); + y1 = _mm256_blend_epi32(y9, y1, 0x55); + y8 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1280)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1312)); + y2 = _mm256_slli_epi64(y9, 32); + y3 = _mm256_srli_epi64(y8, 32); + y2 = _mm256_blend_epi32(y8, y2, 0xaa); + y3 = _mm256_blend_epi32(y9, y3, 0x55); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1344)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1376)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1408)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1440)); + y0 = _mm256_unpacklo_epi32(y8, y1); + y1 = _mm256_unpackhi_epi32(y8, y1); + y2 = _mm256_unpacklo_epi32(y9, y3); + y3 = _mm256_unpackhi_epi32(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 8: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1568)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 16: 2/2 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1600)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1632)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1664)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1696)); + y8 = _mm256_sub_epi16(y0, y1); + y9 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_add_epi16(y0, y1); + y2 = _mm256_add_epi16(y2, y3); + y1 = _mm256_mullo_epi16(y8, y12); + y3 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y1 = _mm256_mulhi_epi16(y1, y14); + y3 = _mm256_mulhi_epi16(y3, y14); + y1 = _mm256_sub_epi16(y8, y1); + y3 = _mm256_sub_epi16(y9, y3); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1728)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1760)); + y8 = _mm256_add_epi16(y0, y2); + y9 = _mm256_add_epi16(y1, y3); + y2 = _mm256_sub_epi16(y0, y2); + y3 = _mm256_sub_epi16(y1, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y1 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y9, y1); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + /* 2: 2/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1792)); + y9 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1824)); + y4 = _mm256_slli_epi64(y9, 32); + y5 = _mm256_srli_epi64(y8, 32); + y4 = _mm256_blend_epi32(y8, y4, 0xaa); + y5 = _mm256_blend_epi32(y9, y5, 0x55); + y8 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1856)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1888)); + y6 = _mm256_slli_epi64(y9, 32); + y7 = _mm256_srli_epi64(y8, 32); + y6 = _mm256_blend_epi32(y8, y6, 0xaa); + y7 = _mm256_blend_epi32(y9, y7, 0x55); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1920)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1952)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1984)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2016)); + y4 = _mm256_unpacklo_epi32(y8, y5); + y5 = _mm256_unpackhi_epi32(y8, y5); + y6 = _mm256_unpacklo_epi32(y9, y7); + y7 = _mm256_unpackhi_epi32(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 8: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2048)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2080)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2112)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2144)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 16: 2/2 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2176)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2208)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2240)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2272)); + y8 = _mm256_sub_epi16(y4, y5); + y9 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_add_epi16(y4, y5); + y6 = _mm256_add_epi16(y6, y7); + y5 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y5 = _mm256_mulhi_epi16(y5, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y5 = _mm256_sub_epi16(y8, y5); + y7 = _mm256_sub_epi16(y9, y7); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2304)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2336)); + y8 = _mm256_add_epi16(y4, y6); + y9 = _mm256_add_epi16(y5, y7); + y6 = _mm256_sub_epi16(y4, y6); + y7 = _mm256_sub_epi16(y5, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y5 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y5 = _mm256_srai_epi16(y5, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y5 = _mm256_mullo_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + /* 64: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2400)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_sub_epi16(y2, y6); + y9 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_add_epi16(y2, y6); + y3 = _mm256_add_epi16(y3, y7); + y6 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y6 = _mm256_sub_epi16(y8, y6); + y7 = _mm256_sub_epi16(y9, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y3); + /* 128 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2432)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2464)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2496)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2528)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_add_epi16(y2, y6); + y9 = _mm256_add_epi16(y3, y7); + y6 = _mm256_sub_epi16(y2, y6); + y7 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y8, y15); + y3 = _mm256_mulhi_epi16(y9, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y8, y2); + y3 = _mm256_sub_epi16(y9, y3); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y0 = _mm256_mulhi_epi16(y0, y11); + y1 = _mm256_mulhi_epi16(y1, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y0, y8); + y1 = _mm256_sub_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y2, y13); + y9 = _mm256_mullo_epi16(y3, y13); + y2 = _mm256_mulhi_epi16(y2, y11); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + y8 = _mm256_mullo_epi16(y4, y13); + y9 = _mm256_mullo_epi16(y5, y13); + y4 = _mm256_mulhi_epi16(y4, y11); + y5 = _mm256_mulhi_epi16(y5, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y4 = _mm256_sub_epi16(y4, y8); + y5 = _mm256_sub_epi16(y5, y9); + y8 = _mm256_mullo_epi16(y6, y13); + y9 = _mm256_mullo_epi16(y7, y13); + y6 = _mm256_mulhi_epi16(y6, y11); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_add_epi16(y2, y6); + y9 = _mm256_add_epi16(y3, y7); + y6 = _mm256_sub_epi16(y2, y6); + y7 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y8, y15); + y3 = _mm256_mulhi_epi16(y9, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y8, y2); + y3 = _mm256_sub_epi16(y9, y3); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y0 = _mm256_mulhi_epi16(y0, y11); + y1 = _mm256_mulhi_epi16(y1, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y0, y8); + y1 = _mm256_sub_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y2, y13); + y9 = _mm256_mullo_epi16(y3, y13); + y2 = _mm256_mulhi_epi16(y2, y11); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + y8 = _mm256_mullo_epi16(y4, y13); + y9 = _mm256_mullo_epi16(y5, y13); + y4 = _mm256_mulhi_epi16(y4, y11); + y5 = _mm256_mulhi_epi16(y5, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y4 = _mm256_sub_epi16(y4, y8); + y5 = _mm256_sub_epi16(y5, y9); + y8 = _mm256_mullo_epi16(y6, y13); + y9 = _mm256_mullo_epi16(y7, y13); + y6 = _mm256_mulhi_epi16(y6, y11); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y7); + /* Add Errors */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 32)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 96)); + y4 = _mm256_add_epi16(y0, y4); + y5 = _mm256_add_epi16(y1, y5); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_add_epi16(y2, y6); + y7 = _mm256_add_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 224)); + y4 = _mm256_add_epi16(y0, y4); + y5 = _mm256_add_epi16(y1, y5); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_add_epi16(y2, y6); + y7 = _mm256_add_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 352)); + y4 = _mm256_add_epi16(y0, y4); + y5 = _mm256_add_epi16(y1, y5); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_add_epi16(y2, y6); + y7 = _mm256_add_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 480)); + y4 = _mm256_add_epi16(y0, y4); + y5 = _mm256_add_epi16(y1, y5); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_add_epi16(y2, y6); + y7 = _mm256_add_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y3); + r9 = (word64)(r9 + 0x200); + rsi = (word64)(rsi + 0x200); + r12 = (word64)(r12 - 1); + if ((sword64)(r12) > (sword64)(0)) { + goto L_mlkem_encapsulate_avx2_calc; + } + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_qinv, 0)); + /* Pointwise acc mont */ + r13 = (word64)((word64)(sword64)(sword32)(word32)r11); + /* Base mul mont */ + rbx = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), y1); + rdi = (word64)(rdi + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 2); + if ((r13) == (0)) { + goto L_pointwise_acc_mont_end_encap_v; + } +L_pointwise_acc_mont_start_encap_v: + /* Base mul mont add */ + rbx = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), y1); + rdi = (word64)(rdi + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 1); + if ((sword64)(r13) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_encap_v; + } +L_pointwise_acc_mont_end_encap_v: + /* Base mul mont add */ + rbx = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r8, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), y1); + rdi = (word64)(rdi + 0x200); + /* invntt */ + rbx = (word64)((word64)(size_t)L_mlkem_avx2_zetas_inv); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + /* 2: 1/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 0)); + y9 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 32)); + y0 = _mm256_slli_epi64(y9, 32); + y1 = _mm256_srli_epi64(y8, 32); + y0 = _mm256_blend_epi32(y8, y0, 0xaa); + y1 = _mm256_blend_epi32(y9, y1, 0x55); + y8 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 64)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 96)); + y2 = _mm256_slli_epi64(y9, 32); + y3 = _mm256_srli_epi64(y8, 32); + y2 = _mm256_blend_epi32(y8, y2, 0xaa); + y3 = _mm256_blend_epi32(y9, y3, 0x55); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 160)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 192)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 224)); + y0 = _mm256_unpacklo_epi32(y8, y1); + y1 = _mm256_unpackhi_epi32(y8, y1); + y2 = _mm256_unpacklo_epi32(y9, y3); + y3 = _mm256_unpackhi_epi32(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 8: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 352)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 16: 1/2 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 384)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 416)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 448)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 480)); + y8 = _mm256_sub_epi16(y0, y1); + y9 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_add_epi16(y0, y1); + y2 = _mm256_add_epi16(y2, y3); + y1 = _mm256_mullo_epi16(y8, y12); + y3 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y1 = _mm256_mulhi_epi16(y1, y14); + y3 = _mm256_mulhi_epi16(y3, y14); + y1 = _mm256_sub_epi16(y8, y1); + y3 = _mm256_sub_epi16(y9, y3); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 512)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 544)); + y8 = _mm256_add_epi16(y0, y2); + y9 = _mm256_add_epi16(y1, y3); + y2 = _mm256_sub_epi16(y0, y2); + y3 = _mm256_sub_epi16(y1, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y1 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y9, y1); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + /* 2: 1/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 576)); + y9 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 608)); + y4 = _mm256_slli_epi64(y9, 32); + y5 = _mm256_srli_epi64(y8, 32); + y4 = _mm256_blend_epi32(y8, y4, 0xaa); + y5 = _mm256_blend_epi32(y9, y5, 0x55); + y8 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 640)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 672)); + y6 = _mm256_slli_epi64(y9, 32); + y7 = _mm256_srli_epi64(y8, 32); + y6 = _mm256_blend_epi32(y8, y6, 0xaa); + y7 = _mm256_blend_epi32(y9, y7, 0x55); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 704)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 736)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 768)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 800)); + y4 = _mm256_unpacklo_epi32(y8, y5); + y5 = _mm256_unpackhi_epi32(y8, y5); + y6 = _mm256_unpacklo_epi32(y9, y7); + y7 = _mm256_unpackhi_epi32(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 8: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 832)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 864)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 896)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 928)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 16: 1/2 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 960)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 992)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1024)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1056)); + y8 = _mm256_sub_epi16(y4, y5); + y9 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_add_epi16(y4, y5); + y6 = _mm256_add_epi16(y6, y7); + y5 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y5 = _mm256_mulhi_epi16(y5, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y5 = _mm256_sub_epi16(y8, y5); + y7 = _mm256_sub_epi16(y9, y7); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1088)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1120)); + y8 = _mm256_add_epi16(y4, y6); + y9 = _mm256_add_epi16(y5, y7); + y6 = _mm256_sub_epi16(y4, y6); + y7 = _mm256_sub_epi16(y5, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y5 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y5 = _mm256_srai_epi16(y5, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y5 = _mm256_mullo_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + /* 64: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1184)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_sub_epi16(y2, y6); + y9 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_add_epi16(y2, y6); + y3 = _mm256_add_epi16(y3, y7); + y6 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y6 = _mm256_sub_epi16(y8, y6); + y7 = _mm256_sub_epi16(y9, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + /* 2: 2/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1216)); + y9 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1248)); + y0 = _mm256_slli_epi64(y9, 32); + y1 = _mm256_srli_epi64(y8, 32); + y0 = _mm256_blend_epi32(y8, y0, 0xaa); + y1 = _mm256_blend_epi32(y9, y1, 0x55); + y8 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1280)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1312)); + y2 = _mm256_slli_epi64(y9, 32); + y3 = _mm256_srli_epi64(y8, 32); + y2 = _mm256_blend_epi32(y8, y2, 0xaa); + y3 = _mm256_blend_epi32(y9, y3, 0x55); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1344)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1376)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1408)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1440)); + y0 = _mm256_unpacklo_epi32(y8, y1); + y1 = _mm256_unpackhi_epi32(y8, y1); + y2 = _mm256_unpacklo_epi32(y9, y3); + y3 = _mm256_unpackhi_epi32(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 8: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1568)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 16: 2/2 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1600)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1632)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1664)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1696)); + y8 = _mm256_sub_epi16(y0, y1); + y9 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_add_epi16(y0, y1); + y2 = _mm256_add_epi16(y2, y3); + y1 = _mm256_mullo_epi16(y8, y12); + y3 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y1 = _mm256_mulhi_epi16(y1, y14); + y3 = _mm256_mulhi_epi16(y3, y14); + y1 = _mm256_sub_epi16(y8, y1); + y3 = _mm256_sub_epi16(y9, y3); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1728)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1760)); + y8 = _mm256_add_epi16(y0, y2); + y9 = _mm256_add_epi16(y1, y3); + y2 = _mm256_sub_epi16(y0, y2); + y3 = _mm256_sub_epi16(y1, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y1 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y9, y1); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + /* 2: 2/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1792)); + y9 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1824)); + y4 = _mm256_slli_epi64(y9, 32); + y5 = _mm256_srli_epi64(y8, 32); + y4 = _mm256_blend_epi32(y8, y4, 0xaa); + y5 = _mm256_blend_epi32(y9, y5, 0x55); + y8 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1856)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1888)); + y6 = _mm256_slli_epi64(y9, 32); + y7 = _mm256_srli_epi64(y8, 32); + y6 = _mm256_blend_epi32(y8, y6, 0xaa); + y7 = _mm256_blend_epi32(y9, y7, 0x55); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1920)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1952)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 1984)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2016)); + y4 = _mm256_unpacklo_epi32(y8, y5); + y5 = _mm256_unpackhi_epi32(y8, y5); + y6 = _mm256_unpacklo_epi32(y9, y7); + y7 = _mm256_unpackhi_epi32(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 8: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2048)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2080)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2112)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2144)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 16: 2/2 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2176)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2208)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2240)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2272)); + y8 = _mm256_sub_epi16(y4, y5); + y9 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_add_epi16(y4, y5); + y6 = _mm256_add_epi16(y6, y7); + y5 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y5 = _mm256_mulhi_epi16(y5, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y5 = _mm256_sub_epi16(y8, y5); + y7 = _mm256_sub_epi16(y9, y7); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2304)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2336)); + y8 = _mm256_add_epi16(y4, y6); + y9 = _mm256_add_epi16(y5, y7); + y6 = _mm256_sub_epi16(y4, y6); + y7 = _mm256_sub_epi16(y5, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y5 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y5 = _mm256_srai_epi16(y5, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y5 = _mm256_mullo_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + /* 64: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2400)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_sub_epi16(y2, y6); + y9 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_add_epi16(y2, y6); + y3 = _mm256_add_epi16(y3, y7); + y6 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y6 = _mm256_sub_epi16(y8, y6); + y7 = _mm256_sub_epi16(y9, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y3); + /* 128 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2432)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2464)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2496)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(rbx, 2528)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_add_epi16(y2, y6); + y9 = _mm256_add_epi16(y3, y7); + y6 = _mm256_sub_epi16(y2, y6); + y7 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y8, y15); + y3 = _mm256_mulhi_epi16(y9, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y8, y2); + y3 = _mm256_sub_epi16(y9, y3); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y0 = _mm256_mulhi_epi16(y0, y11); + y1 = _mm256_mulhi_epi16(y1, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y0, y8); + y1 = _mm256_sub_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y2, y13); + y9 = _mm256_mullo_epi16(y3, y13); + y2 = _mm256_mulhi_epi16(y2, y11); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + y8 = _mm256_mullo_epi16(y4, y13); + y9 = _mm256_mullo_epi16(y5, y13); + y4 = _mm256_mulhi_epi16(y4, y11); + y5 = _mm256_mulhi_epi16(y5, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y4 = _mm256_sub_epi16(y4, y8); + y5 = _mm256_sub_epi16(y5, y9); + y8 = _mm256_mullo_epi16(y6, y13); + y9 = _mm256_mullo_epi16(y7, y13); + y6 = _mm256_mulhi_epi16(y6, y11); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_add_epi16(y2, y6); + y9 = _mm256_add_epi16(y3, y7); + y6 = _mm256_sub_epi16(y2, y6); + y7 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y8, y15); + y3 = _mm256_mulhi_epi16(y9, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y8, y2); + y3 = _mm256_sub_epi16(y9, y3); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y0 = _mm256_mulhi_epi16(y0, y11); + y1 = _mm256_mulhi_epi16(y1, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y0, y8); + y1 = _mm256_sub_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y2, y13); + y9 = _mm256_mullo_epi16(y3, y13); + y2 = _mm256_mulhi_epi16(y2, y11); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + y8 = _mm256_mullo_epi16(y4, y13); + y9 = _mm256_mullo_epi16(y5, y13); + y4 = _mm256_mulhi_epi16(y4, y11); + y5 = _mm256_mulhi_epi16(y5, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y4 = _mm256_sub_epi16(y4, y8); + y5 = _mm256_sub_epi16(y5, y9); + y8 = _mm256_mullo_epi16(y6, y13); + y9 = _mm256_mullo_epi16(y7, y13); + y6 = _mm256_mulhi_epi16(y6, y11); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y7); + /* Add Errors */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 32)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 96)); + y4 = _mm256_add_epi16(y4, y0); + y5 = _mm256_add_epi16(y5, y1); + y6 = _mm256_add_epi16(y6, y2); + y7 = _mm256_add_epi16(y7, y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y4 = _mm256_add_epi16(y0, y4); + y5 = _mm256_add_epi16(y1, y5); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_add_epi16(y2, y6); + y7 = _mm256_add_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 224)); + y4 = _mm256_add_epi16(y4, y0); + y5 = _mm256_add_epi16(y5, y1); + y6 = _mm256_add_epi16(y6, y2); + y7 = _mm256_add_epi16(y7, y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y4 = _mm256_add_epi16(y0, y4); + y5 = _mm256_add_epi16(y1, y5); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_add_epi16(y2, y6); + y7 = _mm256_add_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 352)); + y4 = _mm256_add_epi16(y4, y0); + y5 = _mm256_add_epi16(y5, y1); + y6 = _mm256_add_epi16(y6, y2); + y7 = _mm256_add_epi16(y7, y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y4 = _mm256_add_epi16(y0, y4); + y5 = _mm256_add_epi16(y1, y5); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_add_epi16(y2, y6); + y7 = _mm256_add_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rax, 480)); + y4 = _mm256_add_epi16(y4, y0); + y5 = _mm256_add_epi16(y5, y1); + y6 = _mm256_add_epi16(y6, y2); + y7 = _mm256_add_epi16(y7, y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y4 = _mm256_add_epi16(y0, y4); + y5 = _mm256_add_epi16(y1, y5); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_add_epi16(y2, y6); + y7 = _mm256_add_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), y3); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_decapsulate_avx2(const sword16* priv, sword16* mp, + sword16* bp, const sword16* vp, int kp) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(), + y10 = _mm256_setzero_si256(), y11 = _mm256_setzero_si256(), + y12 = _mm256_setzero_si256(), y13 = _mm256_setzero_si256(), y14, + y15; + + rdi = (word64)(size_t)priv; + rsi = (word64)(size_t)mp; + rdx = (word64)(size_t)bp; + rcx = (word64)(size_t)vp; + r8 = (word64)(word32)kp; + + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_q, 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_v, 0)); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + r9 = (word64)(rdx); +L_mlkem_decapsulate_avx2_trans: + /* ntt */ + r10 = (word64)((word64)(size_t)L_mlkem_avx2_zetas); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 224)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 480)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 224), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 352)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 352), y7); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 224)); + /* 64: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + /* 16: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 16: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 8: 1/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 512)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 544)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 576)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 608)); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y0 = _mm256_sub_epi16(y1, y0); + y2 = _mm256_sub_epi16(y3, y2); + y1 = _mm256_sub_epi16(y8, y0); + y3 = _mm256_sub_epi16(y9, y2); + y8 = _mm256_add_epi16(y8, y0); + y9 = _mm256_add_epi16(y9, y2); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 640)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 672)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 704)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 736)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 8: 1/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 768)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 800)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 832)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 864)); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y4 = _mm256_sub_epi16(y5, y4); + y6 = _mm256_sub_epi16(y7, y6); + y5 = _mm256_sub_epi16(y8, y4); + y7 = _mm256_sub_epi16(y9, y6); + y8 = _mm256_add_epi16(y8, y4); + y9 = _mm256_add_epi16(y9, y6); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 896)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 928)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 960)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 992)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 2: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1024)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1056)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1088)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1120)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 2: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1184)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1216)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1248)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_mulhi_epi16(y0, y15); + y9 = _mm256_mulhi_epi16(y1, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y0, y8); + y9 = _mm256_sub_epi16(y1, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 0), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 32), y9); + y8 = _mm256_mulhi_epi16(y2, y15); + y9 = _mm256_mulhi_epi16(y3, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 96), y9); + y8 = _mm256_mulhi_epi16(y4, y15); + y9 = _mm256_mulhi_epi16(y5, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 128), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 160), y9); + y8 = _mm256_mulhi_epi16(y6, y15); + y9 = _mm256_mulhi_epi16(y7, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 192), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 224), y9); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(r9, 480)); + /* 64: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1280)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1312)); + y8 = _mm256_mullo_epi16(y4, y12); + y9 = _mm256_mullo_epi16(y5, y12); + y4 = _mm256_mulhi_epi16(y4, y10); + y5 = _mm256_mulhi_epi16(y5, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_sub_epi16(y0, y8); + y5 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y2, y8); + y7 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_add_epi16(y2, y8); + y3 = _mm256_add_epi16(y3, y9); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1344)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1376)); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + y2 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y1, y9); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1408)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1440)); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y5, y9); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + /* 16: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1568)); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 16: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1600)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1632)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1664)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1696)); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 8: 2/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1728)); + y1 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1760)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1792)); + y3 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1824)); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y0 = _mm256_sub_epi16(y1, y0); + y2 = _mm256_sub_epi16(y3, y2); + y1 = _mm256_sub_epi16(y8, y0); + y3 = _mm256_sub_epi16(y9, y2); + y8 = _mm256_add_epi16(y8, y0); + y9 = _mm256_add_epi16(y9, y2); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1856)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1888)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1920)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1952)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 8: 2/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1984)); + y5 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2016)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2048)); + y7 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2080)); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y4 = _mm256_sub_epi16(y5, y4); + y6 = _mm256_sub_epi16(y7, y6); + y5 = _mm256_sub_epi16(y8, y4); + y7 = _mm256_sub_epi16(y9, y6); + y8 = _mm256_add_epi16(y8, y4); + y9 = _mm256_add_epi16(y9, y6); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2112)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2144)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2176)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2208)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + /* 2: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2240)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2272)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2304)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2336)); + y8 = _mm256_slli_epi64(y1, 32); + y9 = _mm256_srli_epi64(y0, 32); + y0 = _mm256_blend_epi32(y0, y8, 0xaa); + y1 = _mm256_blend_epi32(y1, y9, 0x55); + y8 = _mm256_slli_epi64(y3, 32); + y9 = _mm256_srli_epi64(y2, 32); + y2 = _mm256_blend_epi32(y2, y8, 0xaa); + y3 = _mm256_blend_epi32(y3, y9, 0x55); + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y1, y8); + y9 = _mm256_sub_epi16(y3, y9); + y1 = _mm256_sub_epi16(y0, y8); + y3 = _mm256_sub_epi16(y2, y9); + y0 = _mm256_add_epi16(y0, y8); + y2 = _mm256_add_epi16(y2, y9); + /* 2: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2400)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2432)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2464)); + y8 = _mm256_slli_epi64(y5, 32); + y9 = _mm256_srli_epi64(y4, 32); + y4 = _mm256_blend_epi32(y4, y8, 0xaa); + y5 = _mm256_blend_epi32(y5, y9, 0x55); + y8 = _mm256_slli_epi64(y7, 32); + y9 = _mm256_srli_epi64(y6, 32); + y6 = _mm256_blend_epi32(y6, y8, 0xaa); + y7 = _mm256_blend_epi32(y7, y9, 0x55); + y8 = _mm256_mullo_epi16(y5, y12); + y9 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y8 = _mm256_sub_epi16(y5, y8); + y9 = _mm256_sub_epi16(y7, y9); + y5 = _mm256_sub_epi16(y4, y8); + y7 = _mm256_sub_epi16(y6, y9); + y4 = _mm256_add_epi16(y4, y8); + y6 = _mm256_add_epi16(y6, y9); + y8 = _mm256_unpacklo_epi32(y0, y1); + y9 = _mm256_unpackhi_epi32(y0, y1); + y0 = _mm256_permute2x128_si256(y8, y9, 0x20); + y1 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y2, y3); + y9 = _mm256_unpackhi_epi32(y2, y3); + y2 = _mm256_permute2x128_si256(y8, y9, 0x20); + y3 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y4, y5); + y9 = _mm256_unpackhi_epi32(y4, y5); + y4 = _mm256_permute2x128_si256(y8, y9, 0x20); + y5 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_unpacklo_epi32(y6, y7); + y9 = _mm256_unpackhi_epi32(y6, y7); + y6 = _mm256_permute2x128_si256(y8, y9, 0x20); + y7 = _mm256_permute2x128_si256(y8, y9, 0x31); + y8 = _mm256_mulhi_epi16(y0, y15); + y9 = _mm256_mulhi_epi16(y1, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y0, y8); + y9 = _mm256_sub_epi16(y1, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 256), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 288), y9); + y8 = _mm256_mulhi_epi16(y2, y15); + y9 = _mm256_mulhi_epi16(y3, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y2, y8); + y9 = _mm256_sub_epi16(y3, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 320), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 352), y9); + y8 = _mm256_mulhi_epi16(y4, y15); + y9 = _mm256_mulhi_epi16(y5, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y4, y8); + y9 = _mm256_sub_epi16(y5, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 384), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 416), y9); + y8 = _mm256_mulhi_epi16(y6, y15); + y9 = _mm256_mulhi_epi16(y7, y15); + y8 = _mm256_srai_epi16(y8, 10); + y9 = _mm256_srai_epi16(y9, 10); + y8 = _mm256_mullo_epi16(y8, y14); + y9 = _mm256_mullo_epi16(y9, y14); + y8 = _mm256_sub_epi16(y6, y8); + y9 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 448), y8); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 480), y9); + r9 = (word64)(r9 + 0x200); + rax = (word64)(rax - 1); + if ((sword64)(rax) > (sword64)(0)) { + goto L_mlkem_decapsulate_avx2_trans; + } + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_qinv, 0)); + /* Pointwise acc mont */ + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + /* Base mul mont */ + r10 = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y1); + rdi = (word64)(rdi + 0x200); + rdx = (word64)(rdx + 0x200); + rax = (word64)(rax - 2); + if ((rax) == (0)) { + goto L_pointwise_acc_mont_end_decap; + } +L_pointwise_acc_mont_start_decap: + /* Base mul mont add */ + r10 = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y1); + rdi = (word64)(rdi + 0x200); + rdx = (word64)(rdx + 0x200); + rax = (word64)(rax - 1); + if ((sword64)(rax) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_decap; + } +L_pointwise_acc_mont_end_decap: + /* Base mul mont add */ + r10 = (word64)((word64)(size_t)L_mlkem_avx2_zetas_basemul); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 32)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 64)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 96)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 160)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 192)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 224)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 288)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 352)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 416)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y1); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y6 = _mm256_slli_epi32(y3, 16); + y7 = _mm256_srli_epi32(y2, 16); + y2 = _mm256_blend_epi16(y2, y6, 0xaa); + y3 = _mm256_blend_epi16(y3, y7, 0x55); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 448)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdx, 480)); + y6 = _mm256_slli_epi32(y5, 16); + y7 = _mm256_srli_epi32(y4, 16); + y4 = _mm256_blend_epi16(y4, y6, 0xaa); + y5 = _mm256_blend_epi16(y5, y7, 0x55); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y0 = _mm256_mullo_epi16(y3, y5); + y6 = _mm256_mulhi_epi16(y3, y5); + y1 = _mm256_mullo_epi16(y2, y4); + y7 = _mm256_mulhi_epi16(y2, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y0, y12); + y9 = _mm256_mullo_epi16(y1, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y6, y8); + y1 = _mm256_sub_epi16(y7, y9); + y6 = _mm256_mullo_epi16(y0, y11); + y7 = _mm256_mulhi_epi16(y0, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y0 = _mm256_sub_epi16(y7, y6); + y0 = _mm256_add_epi16(y0, y1); + y1 = _mm256_mullo_epi16(y2, y5); + y6 = _mm256_mulhi_epi16(y2, y5); + y2 = _mm256_mullo_epi16(y3, y4); + y7 = _mm256_mulhi_epi16(y3, y4); + /* Mont Reduce */ + y8 = _mm256_mullo_epi16(y1, y12); + y9 = _mm256_mullo_epi16(y2, y12); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y1 = _mm256_sub_epi16(y6, y8); + y2 = _mm256_sub_epi16(y7, y9); + y1 = _mm256_add_epi16(y1, y2); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y0 = _mm256_add_epi16(y0, y6); + y1 = _mm256_add_epi16(y1, y7); + y6 = _mm256_slli_epi32(y1, 16); + y7 = _mm256_srli_epi32(y0, 16); + y0 = _mm256_blend_epi16(y0, y6, 0xaa); + y1 = _mm256_blend_epi16(y1, y7, 0x55); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y1); + rdi = (word64)(rdi + 0x200); + /* invntt */ + r10 = (word64)((word64)(size_t)L_mlkem_avx2_zetas_inv); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + /* 2: 1/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 0)); + y9 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 32)); + y0 = _mm256_slli_epi64(y9, 32); + y1 = _mm256_srli_epi64(y8, 32); + y0 = _mm256_blend_epi32(y8, y0, 0xaa); + y1 = _mm256_blend_epi32(y9, y1, 0x55); + y8 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 64)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 96)); + y2 = _mm256_slli_epi64(y9, 32); + y3 = _mm256_srli_epi64(y8, 32); + y2 = _mm256_blend_epi32(y8, y2, 0xaa); + y3 = _mm256_blend_epi32(y9, y3, 0x55); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 128)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 160)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 192)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 224)); + y0 = _mm256_unpacklo_epi32(y8, y1); + y1 = _mm256_unpackhi_epi32(y8, y1); + y2 = _mm256_unpacklo_epi32(y9, y3); + y3 = _mm256_unpackhi_epi32(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 8: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 256)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 288)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 320)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 352)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 16: 1/2 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 384)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 416)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 448)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 480)); + y8 = _mm256_sub_epi16(y0, y1); + y9 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_add_epi16(y0, y1); + y2 = _mm256_add_epi16(y2, y3); + y1 = _mm256_mullo_epi16(y8, y12); + y3 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y1 = _mm256_mulhi_epi16(y1, y14); + y3 = _mm256_mulhi_epi16(y3, y14); + y1 = _mm256_sub_epi16(y8, y1); + y3 = _mm256_sub_epi16(y9, y3); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 512)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 544)); + y8 = _mm256_add_epi16(y0, y2); + y9 = _mm256_add_epi16(y1, y3); + y2 = _mm256_sub_epi16(y0, y2); + y3 = _mm256_sub_epi16(y1, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y1 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y9, y1); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + /* 2: 1/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 576)); + y9 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 608)); + y4 = _mm256_slli_epi64(y9, 32); + y5 = _mm256_srli_epi64(y8, 32); + y4 = _mm256_blend_epi32(y8, y4, 0xaa); + y5 = _mm256_blend_epi32(y9, y5, 0x55); + y8 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 640)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 672)); + y6 = _mm256_slli_epi64(y9, 32); + y7 = _mm256_srli_epi64(y8, 32); + y6 = _mm256_blend_epi32(y8, y6, 0xaa); + y7 = _mm256_blend_epi32(y9, y7, 0x55); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 4: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 704)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 736)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 768)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 800)); + y4 = _mm256_unpacklo_epi32(y8, y5); + y5 = _mm256_unpackhi_epi32(y8, y5); + y6 = _mm256_unpacklo_epi32(y9, y7); + y7 = _mm256_unpackhi_epi32(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 8: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 832)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 864)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 896)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 928)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 16: 1/2 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 960)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 992)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1024)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1056)); + y8 = _mm256_sub_epi16(y4, y5); + y9 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_add_epi16(y4, y5); + y6 = _mm256_add_epi16(y6, y7); + y5 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y5 = _mm256_mulhi_epi16(y5, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y5 = _mm256_sub_epi16(y8, y5); + y7 = _mm256_sub_epi16(y9, y7); + /* 32: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1088)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1120)); + y8 = _mm256_add_epi16(y4, y6); + y9 = _mm256_add_epi16(y5, y7); + y6 = _mm256_sub_epi16(y4, y6); + y7 = _mm256_sub_epi16(y5, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y5 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y5 = _mm256_srai_epi16(y5, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y5 = _mm256_mullo_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + /* 64: 1/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1152)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1184)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_sub_epi16(y2, y6); + y9 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_add_epi16(y2, y6); + y3 = _mm256_add_epi16(y3, y7); + y6 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y6 = _mm256_sub_epi16(y8, y6); + y7 = _mm256_sub_epi16(y9, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + /* 2: 2/2 */ + y8 = _mm256_permute2x128_si256(y0, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1216)); + y9 = _mm256_permute2x128_si256(y0, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1248)); + y0 = _mm256_slli_epi64(y9, 32); + y1 = _mm256_srli_epi64(y8, 32); + y0 = _mm256_blend_epi32(y8, y0, 0xaa); + y1 = _mm256_blend_epi32(y9, y1, 0x55); + y8 = _mm256_permute2x128_si256(y2, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1280)); + y9 = _mm256_permute2x128_si256(y2, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1312)); + y2 = _mm256_slli_epi64(y9, 32); + y3 = _mm256_srli_epi64(y8, 32); + y2 = _mm256_blend_epi32(y8, y2, 0xaa); + y3 = _mm256_blend_epi32(y9, y3, 0x55); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1344)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1376)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1408)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1440)); + y0 = _mm256_unpacklo_epi32(y8, y1); + y1 = _mm256_unpackhi_epi32(y8, y1); + y2 = _mm256_unpacklo_epi32(y9, y3); + y3 = _mm256_unpackhi_epi32(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 8: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1472)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1504)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1536)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1568)); + y0 = _mm256_unpacklo_epi64(y8, y1); + y1 = _mm256_unpackhi_epi64(y8, y1); + y2 = _mm256_unpacklo_epi64(y9, y3); + y3 = _mm256_unpackhi_epi64(y9, y3); + y8 = _mm256_add_epi16(y0, y1); + y9 = _mm256_add_epi16(y2, y3); + y1 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y2 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y2 = _mm256_srai_epi16(y2, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y2 = _mm256_mullo_epi16(y2, y14); + y8 = _mm256_sub_epi16(y8, y0); + y9 = _mm256_sub_epi16(y9, y2); + y0 = _mm256_mullo_epi16(y1, y12); + y2 = _mm256_mullo_epi16(y3, y13); + y1 = _mm256_mulhi_epi16(y1, y10); + y3 = _mm256_mulhi_epi16(y3, y11); + y0 = _mm256_mulhi_epi16(y0, y14); + y2 = _mm256_mulhi_epi16(y2, y14); + y1 = _mm256_sub_epi16(y1, y0); + y3 = _mm256_sub_epi16(y3, y2); + /* 16: 2/2 */ + y0 = _mm256_permute2x128_si256(y8, y1, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1600)); + y1 = _mm256_permute2x128_si256(y8, y1, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1632)); + y2 = _mm256_permute2x128_si256(y9, y3, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1664)); + y3 = _mm256_permute2x128_si256(y9, y3, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1696)); + y8 = _mm256_sub_epi16(y0, y1); + y9 = _mm256_sub_epi16(y2, y3); + y0 = _mm256_add_epi16(y0, y1); + y2 = _mm256_add_epi16(y2, y3); + y1 = _mm256_mullo_epi16(y8, y12); + y3 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y1 = _mm256_mulhi_epi16(y1, y14); + y3 = _mm256_mulhi_epi16(y3, y14); + y1 = _mm256_sub_epi16(y8, y1); + y3 = _mm256_sub_epi16(y9, y3); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1728)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1760)); + y8 = _mm256_add_epi16(y0, y2); + y9 = _mm256_add_epi16(y1, y3); + y2 = _mm256_sub_epi16(y0, y2); + y3 = _mm256_sub_epi16(y1, y3); + y0 = _mm256_mulhi_epi16(y8, y15); + y1 = _mm256_mulhi_epi16(y9, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y8, y0); + y1 = _mm256_sub_epi16(y9, y1); + y8 = _mm256_mullo_epi16(y2, y12); + y9 = _mm256_mullo_epi16(y3, y12); + y2 = _mm256_mulhi_epi16(y2, y10); + y3 = _mm256_mulhi_epi16(y3, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + /* 2: 2/2 */ + y8 = _mm256_permute2x128_si256(y4, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1792)); + y9 = _mm256_permute2x128_si256(y4, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1824)); + y4 = _mm256_slli_epi64(y9, 32); + y5 = _mm256_srli_epi64(y8, 32); + y4 = _mm256_blend_epi32(y8, y4, 0xaa); + y5 = _mm256_blend_epi32(y9, y5, 0x55); + y8 = _mm256_permute2x128_si256(y6, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1856)); + y9 = _mm256_permute2x128_si256(y6, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1888)); + y6 = _mm256_slli_epi64(y9, 32); + y7 = _mm256_srli_epi64(y8, 32); + y6 = _mm256_blend_epi32(y8, y6, 0xaa); + y7 = _mm256_blend_epi32(y9, y7, 0x55); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 4: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1920)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1952)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 1984)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2016)); + y4 = _mm256_unpacklo_epi32(y8, y5); + y5 = _mm256_unpackhi_epi32(y8, y5); + y6 = _mm256_unpacklo_epi32(y9, y7); + y7 = _mm256_unpackhi_epi32(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 8: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2048)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2080)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2112)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2144)); + y4 = _mm256_unpacklo_epi64(y8, y5); + y5 = _mm256_unpackhi_epi64(y8, y5); + y6 = _mm256_unpacklo_epi64(y9, y7); + y7 = _mm256_unpackhi_epi64(y9, y7); + y8 = _mm256_add_epi16(y4, y5); + y9 = _mm256_add_epi16(y6, y7); + y5 = _mm256_sub_epi16(y4, y5); + y7 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y6 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y6 = _mm256_srai_epi16(y6, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y6 = _mm256_mullo_epi16(y6, y14); + y8 = _mm256_sub_epi16(y8, y4); + y9 = _mm256_sub_epi16(y9, y6); + y4 = _mm256_mullo_epi16(y5, y12); + y6 = _mm256_mullo_epi16(y7, y13); + y5 = _mm256_mulhi_epi16(y5, y10); + y7 = _mm256_mulhi_epi16(y7, y11); + y4 = _mm256_mulhi_epi16(y4, y14); + y6 = _mm256_mulhi_epi16(y6, y14); + y5 = _mm256_sub_epi16(y5, y4); + y7 = _mm256_sub_epi16(y7, y6); + /* 16: 2/2 */ + y4 = _mm256_permute2x128_si256(y8, y5, 0x20); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2176)); + y5 = _mm256_permute2x128_si256(y8, y5, 0x31); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2208)); + y6 = _mm256_permute2x128_si256(y9, y7, 0x20); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2240)); + y7 = _mm256_permute2x128_si256(y9, y7, 0x31); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2272)); + y8 = _mm256_sub_epi16(y4, y5); + y9 = _mm256_sub_epi16(y6, y7); + y4 = _mm256_add_epi16(y4, y5); + y6 = _mm256_add_epi16(y6, y7); + y5 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y13); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y11); + y5 = _mm256_mulhi_epi16(y5, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y5 = _mm256_sub_epi16(y8, y5); + y7 = _mm256_sub_epi16(y9, y7); + /* 32: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2304)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2336)); + y8 = _mm256_add_epi16(y4, y6); + y9 = _mm256_add_epi16(y5, y7); + y6 = _mm256_sub_epi16(y4, y6); + y7 = _mm256_sub_epi16(y5, y7); + y4 = _mm256_mulhi_epi16(y8, y15); + y5 = _mm256_mulhi_epi16(y9, y15); + y4 = _mm256_srai_epi16(y4, 10); + y5 = _mm256_srai_epi16(y5, 10); + y4 = _mm256_mullo_epi16(y4, y14); + y5 = _mm256_mullo_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + /* 64: 2/2 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2368)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2400)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_sub_epi16(y2, y6); + y9 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_add_epi16(y2, y6); + y3 = _mm256_add_epi16(y3, y7); + y6 = _mm256_mullo_epi16(y8, y12); + y7 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y6 = _mm256_mulhi_epi16(y6, y14); + y7 = _mm256_mulhi_epi16(y7, y14); + y6 = _mm256_sub_epi16(y8, y6); + y7 = _mm256_sub_epi16(y9, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y3); + /* 128 */ + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2432)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2464)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2496)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(r10, 2528)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_add_epi16(y2, y6); + y9 = _mm256_add_epi16(y3, y7); + y6 = _mm256_sub_epi16(y2, y6); + y7 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y8, y15); + y3 = _mm256_mulhi_epi16(y9, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y8, y2); + y3 = _mm256_sub_epi16(y9, y3); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y0 = _mm256_mulhi_epi16(y0, y11); + y1 = _mm256_mulhi_epi16(y1, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y0, y8); + y1 = _mm256_sub_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y2, y13); + y9 = _mm256_mullo_epi16(y3, y13); + y2 = _mm256_mulhi_epi16(y2, y11); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + y8 = _mm256_mullo_epi16(y4, y13); + y9 = _mm256_mullo_epi16(y5, y13); + y4 = _mm256_mulhi_epi16(y4, y11); + y5 = _mm256_mulhi_epi16(y5, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y4 = _mm256_sub_epi16(y4, y8); + y5 = _mm256_sub_epi16(y5, y9); + y8 = _mm256_mullo_epi16(y6, y13); + y9 = _mm256_mullo_epi16(y7, y13); + y6 = _mm256_mulhi_epi16(y6, y11); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y8 = _mm256_sub_epi16(y0, y4); + y9 = _mm256_sub_epi16(y1, y5); + y0 = _mm256_add_epi16(y0, y4); + y1 = _mm256_add_epi16(y1, y5); + y4 = _mm256_mullo_epi16(y8, y12); + y5 = _mm256_mullo_epi16(y9, y12); + y8 = _mm256_mulhi_epi16(y8, y10); + y9 = _mm256_mulhi_epi16(y9, y10); + y4 = _mm256_mulhi_epi16(y4, y14); + y5 = _mm256_mulhi_epi16(y5, y14); + y4 = _mm256_sub_epi16(y8, y4); + y5 = _mm256_sub_epi16(y9, y5); + y8 = _mm256_add_epi16(y2, y6); + y9 = _mm256_add_epi16(y3, y7); + y6 = _mm256_sub_epi16(y2, y6); + y7 = _mm256_sub_epi16(y3, y7); + y2 = _mm256_mulhi_epi16(y8, y15); + y3 = _mm256_mulhi_epi16(y9, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y8, y2); + y3 = _mm256_sub_epi16(y9, y3); + y8 = _mm256_mullo_epi16(y6, y12); + y9 = _mm256_mullo_epi16(y7, y12); + y6 = _mm256_mulhi_epi16(y6, y10); + y7 = _mm256_mulhi_epi16(y7, y10); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + y8 = _mm256_mullo_epi16(y0, y13); + y9 = _mm256_mullo_epi16(y1, y13); + y0 = _mm256_mulhi_epi16(y0, y11); + y1 = _mm256_mulhi_epi16(y1, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y0 = _mm256_sub_epi16(y0, y8); + y1 = _mm256_sub_epi16(y1, y9); + y8 = _mm256_mullo_epi16(y2, y13); + y9 = _mm256_mullo_epi16(y3, y13); + y2 = _mm256_mulhi_epi16(y2, y11); + y3 = _mm256_mulhi_epi16(y3, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y2 = _mm256_sub_epi16(y2, y8); + y3 = _mm256_sub_epi16(y3, y9); + y8 = _mm256_mullo_epi16(y4, y13); + y9 = _mm256_mullo_epi16(y5, y13); + y4 = _mm256_mulhi_epi16(y4, y11); + y5 = _mm256_mulhi_epi16(y5, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y4 = _mm256_sub_epi16(y4, y8); + y5 = _mm256_sub_epi16(y5, y9); + y8 = _mm256_mullo_epi16(y6, y13); + y9 = _mm256_mullo_epi16(y7, y13); + y6 = _mm256_mulhi_epi16(y6, y11); + y7 = _mm256_mulhi_epi16(y7, y11); + y8 = _mm256_mulhi_epi16(y8, y14); + y9 = _mm256_mulhi_epi16(y9, y14); + y6 = _mm256_sub_epi16(y6, y8); + y7 = _mm256_sub_epi16(y7, y9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y7); + /* Sub Errors */ + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 32)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 64)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 96)); + y4 = _mm256_sub_epi16(y4, y0); + y5 = _mm256_sub_epi16(y5, y1); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_sub_epi16(y6, y2); + y7 = _mm256_sub_epi16(y7, y3); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 224)); + y4 = _mm256_sub_epi16(y4, y0); + y5 = _mm256_sub_epi16(y5, y1); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_sub_epi16(y6, y2); + y7 = _mm256_sub_epi16(y7, y3); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 256)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 288)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 320)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 352)); + y4 = _mm256_sub_epi16(y4, y0); + y5 = _mm256_sub_epi16(y5, y1); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_sub_epi16(y6, y2); + y7 = _mm256_sub_epi16(y7, y3); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rcx, 480)); + y4 = _mm256_sub_epi16(y4, y0); + y5 = _mm256_sub_epi16(y5, y1); + y0 = _mm256_mulhi_epi16(y4, y15); + y1 = _mm256_mulhi_epi16(y5, y15); + y0 = _mm256_srai_epi16(y0, 10); + y1 = _mm256_srai_epi16(y1, 10); + y0 = _mm256_mullo_epi16(y0, y14); + y1 = _mm256_mullo_epi16(y1, y14); + y0 = _mm256_sub_epi16(y4, y0); + y1 = _mm256_sub_epi16(y5, y1); + y6 = _mm256_sub_epi16(y6, y2); + y7 = _mm256_sub_epi16(y7, y3); + y2 = _mm256_mulhi_epi16(y6, y15); + y3 = _mm256_mulhi_epi16(y7, y15); + y2 = _mm256_srai_epi16(y2, 10); + y3 = _mm256_srai_epi16(y3, 10); + y2 = _mm256_mullo_epi16(y2, y14); + y3 = _mm256_mullo_epi16(y3, y14); + y2 = _mm256_sub_epi16(y6, y2); + y3 = _mm256_sub_epi16(y7, y3); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), y3); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_csubq_avx2(sword16* p) +{ + word64 rdi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12; + + rdi = (word64)(size_t)p; + + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_q, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_sub_epi16(y0, y12); + y9 = _mm256_sub_epi16(y1, y12); + y10 = _mm256_sub_epi16(y2, y12); + y11 = _mm256_sub_epi16(y3, y12); + y0 = _mm256_srai_epi16(y8, 15); + y1 = _mm256_srai_epi16(y9, 15); + y2 = _mm256_srai_epi16(y10, 15); + y3 = _mm256_srai_epi16(y11, 15); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y2 = _mm256_add_epi16(y2, y10); + y3 = _mm256_add_epi16(y3, y11); + y8 = _mm256_sub_epi16(y4, y12); + y9 = _mm256_sub_epi16(y5, y12); + y10 = _mm256_sub_epi16(y6, y12); + y11 = _mm256_sub_epi16(y7, y12); + y4 = _mm256_srai_epi16(y8, 15); + y5 = _mm256_srai_epi16(y9, 15); + y6 = _mm256_srai_epi16(y10, 15); + y7 = _mm256_srai_epi16(y11, 15); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + y6 = _mm256_and_si256(y6, y12); + y7 = _mm256_and_si256(y7, y12); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + y6 = _mm256_add_epi16(y6, y10); + y7 = _mm256_add_epi16(y7, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y8 = _mm256_sub_epi16(y0, y12); + y9 = _mm256_sub_epi16(y1, y12); + y10 = _mm256_sub_epi16(y2, y12); + y11 = _mm256_sub_epi16(y3, y12); + y0 = _mm256_srai_epi16(y8, 15); + y1 = _mm256_srai_epi16(y9, 15); + y2 = _mm256_srai_epi16(y10, 15); + y3 = _mm256_srai_epi16(y11, 15); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y2 = _mm256_add_epi16(y2, y10); + y3 = _mm256_add_epi16(y3, y11); + y8 = _mm256_sub_epi16(y4, y12); + y9 = _mm256_sub_epi16(y5, y12); + y10 = _mm256_sub_epi16(y6, y12); + y11 = _mm256_sub_epi16(y7, y12); + y4 = _mm256_srai_epi16(y8, 15); + y5 = _mm256_srai_epi16(y9, 15); + y6 = _mm256_srai_epi16(y10, 15); + y7 = _mm256_srai_epi16(y11, 15); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + y6 = _mm256_and_si256(y6, y12); + y7 = _mm256_and_si256(y7, y12); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + y6 = _mm256_add_epi16(y6, y10); + y7 = _mm256_add_epi16(y7, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); +} + +XALIGNED(32) static const word64 L_mlkem_rej_idx[] WC_X64I_UNUSED = { + 0xffffffffffffffffULL, 0xffffffffffffff00ULL, + 0xffffffffffffff02ULL, 0xffffffffffff0200ULL, + 0xffffffffffffff04ULL, 0xffffffffffff0400ULL, + 0xffffffffffff0402ULL, 0xffffffffff040200ULL, + 0xffffffffffffff06ULL, 0xffffffffffff0600ULL, + 0xffffffffffff0602ULL, 0xffffffffff060200ULL, + 0xffffffffffff0604ULL, 0xffffffffff060400ULL, + 0xffffffffff060402ULL, 0xffffffff06040200ULL, + 0xffffffffffffff08ULL, 0xffffffffffff0800ULL, + 0xffffffffffff0802ULL, 0xffffffffff080200ULL, + 0xffffffffffff0804ULL, 0xffffffffff080400ULL, + 0xffffffffff080402ULL, 0xffffffff08040200ULL, + 0xffffffffffff0806ULL, 0xffffffffff080600ULL, + 0xffffffffff080602ULL, 0xffffffff08060200ULL, + 0xffffffffff080604ULL, 0xffffffff08060400ULL, + 0xffffffff08060402ULL, 0xffffff0806040200ULL, + 0xffffffffffffff0aULL, 0xffffffffffff0a00ULL, + 0xffffffffffff0a02ULL, 0xffffffffff0a0200ULL, + 0xffffffffffff0a04ULL, 0xffffffffff0a0400ULL, + 0xffffffffff0a0402ULL, 0xffffffff0a040200ULL, + 0xffffffffffff0a06ULL, 0xffffffffff0a0600ULL, + 0xffffffffff0a0602ULL, 0xffffffff0a060200ULL, + 0xffffffffff0a0604ULL, 0xffffffff0a060400ULL, + 0xffffffff0a060402ULL, 0xffffff0a06040200ULL, + 0xffffffffffff0a08ULL, 0xffffffffff0a0800ULL, + 0xffffffffff0a0802ULL, 0xffffffff0a080200ULL, + 0xffffffffff0a0804ULL, 0xffffffff0a080400ULL, + 0xffffffff0a080402ULL, 0xffffff0a08040200ULL, + 0xffffffffff0a0806ULL, 0xffffffff0a080600ULL, + 0xffffffff0a080602ULL, 0xffffff0a08060200ULL, + 0xffffffff0a080604ULL, 0xffffff0a08060400ULL, + 0xffffff0a08060402ULL, 0xffff0a0806040200ULL, + 0xffffffffffffff0cULL, 0xffffffffffff0c00ULL, + 0xffffffffffff0c02ULL, 0xffffffffff0c0200ULL, + 0xffffffffffff0c04ULL, 0xffffffffff0c0400ULL, + 0xffffffffff0c0402ULL, 0xffffffff0c040200ULL, + 0xffffffffffff0c06ULL, 0xffffffffff0c0600ULL, + 0xffffffffff0c0602ULL, 0xffffffff0c060200ULL, + 0xffffffffff0c0604ULL, 0xffffffff0c060400ULL, + 0xffffffff0c060402ULL, 0xffffff0c06040200ULL, + 0xffffffffffff0c08ULL, 0xffffffffff0c0800ULL, + 0xffffffffff0c0802ULL, 0xffffffff0c080200ULL, + 0xffffffffff0c0804ULL, 0xffffffff0c080400ULL, + 0xffffffff0c080402ULL, 0xffffff0c08040200ULL, + 0xffffffffff0c0806ULL, 0xffffffff0c080600ULL, + 0xffffffff0c080602ULL, 0xffffff0c08060200ULL, + 0xffffffff0c080604ULL, 0xffffff0c08060400ULL, + 0xffffff0c08060402ULL, 0xffff0c0806040200ULL, + 0xffffffffffff0c0aULL, 0xffffffffff0c0a00ULL, + 0xffffffffff0c0a02ULL, 0xffffffff0c0a0200ULL, + 0xffffffffff0c0a04ULL, 0xffffffff0c0a0400ULL, + 0xffffffff0c0a0402ULL, 0xffffff0c0a040200ULL, + 0xffffffffff0c0a06ULL, 0xffffffff0c0a0600ULL, + 0xffffffff0c0a0602ULL, 0xffffff0c0a060200ULL, + 0xffffffff0c0a0604ULL, 0xffffff0c0a060400ULL, + 0xffffff0c0a060402ULL, 0xffff0c0a06040200ULL, + 0xffffffffff0c0a08ULL, 0xffffffff0c0a0800ULL, + 0xffffffff0c0a0802ULL, 0xffffff0c0a080200ULL, + 0xffffffff0c0a0804ULL, 0xffffff0c0a080400ULL, + 0xffffff0c0a080402ULL, 0xffff0c0a08040200ULL, + 0xffffffff0c0a0806ULL, 0xffffff0c0a080600ULL, + 0xffffff0c0a080602ULL, 0xffff0c0a08060200ULL, + 0xffffff0c0a080604ULL, 0xffff0c0a08060400ULL, + 0xffff0c0a08060402ULL, 0xff0c0a0806040200ULL, + 0xffffffffffffff0eULL, 0xffffffffffff0e00ULL, + 0xffffffffffff0e02ULL, 0xffffffffff0e0200ULL, + 0xffffffffffff0e04ULL, 0xffffffffff0e0400ULL, + 0xffffffffff0e0402ULL, 0xffffffff0e040200ULL, + 0xffffffffffff0e06ULL, 0xffffffffff0e0600ULL, + 0xffffffffff0e0602ULL, 0xffffffff0e060200ULL, + 0xffffffffff0e0604ULL, 0xffffffff0e060400ULL, + 0xffffffff0e060402ULL, 0xffffff0e06040200ULL, + 0xffffffffffff0e08ULL, 0xffffffffff0e0800ULL, + 0xffffffffff0e0802ULL, 0xffffffff0e080200ULL, + 0xffffffffff0e0804ULL, 0xffffffff0e080400ULL, + 0xffffffff0e080402ULL, 0xffffff0e08040200ULL, + 0xffffffffff0e0806ULL, 0xffffffff0e080600ULL, + 0xffffffff0e080602ULL, 0xffffff0e08060200ULL, + 0xffffffff0e080604ULL, 0xffffff0e08060400ULL, + 0xffffff0e08060402ULL, 0xffff0e0806040200ULL, + 0xffffffffffff0e0aULL, 0xffffffffff0e0a00ULL, + 0xffffffffff0e0a02ULL, 0xffffffff0e0a0200ULL, + 0xffffffffff0e0a04ULL, 0xffffffff0e0a0400ULL, + 0xffffffff0e0a0402ULL, 0xffffff0e0a040200ULL, + 0xffffffffff0e0a06ULL, 0xffffffff0e0a0600ULL, + 0xffffffff0e0a0602ULL, 0xffffff0e0a060200ULL, + 0xffffffff0e0a0604ULL, 0xffffff0e0a060400ULL, + 0xffffff0e0a060402ULL, 0xffff0e0a06040200ULL, + 0xffffffffff0e0a08ULL, 0xffffffff0e0a0800ULL, + 0xffffffff0e0a0802ULL, 0xffffff0e0a080200ULL, + 0xffffffff0e0a0804ULL, 0xffffff0e0a080400ULL, + 0xffffff0e0a080402ULL, 0xffff0e0a08040200ULL, + 0xffffffff0e0a0806ULL, 0xffffff0e0a080600ULL, + 0xffffff0e0a080602ULL, 0xffff0e0a08060200ULL, + 0xffffff0e0a080604ULL, 0xffff0e0a08060400ULL, + 0xffff0e0a08060402ULL, 0xff0e0a0806040200ULL, + 0xffffffffffff0e0cULL, 0xffffffffff0e0c00ULL, + 0xffffffffff0e0c02ULL, 0xffffffff0e0c0200ULL, + 0xffffffffff0e0c04ULL, 0xffffffff0e0c0400ULL, + 0xffffffff0e0c0402ULL, 0xffffff0e0c040200ULL, + 0xffffffffff0e0c06ULL, 0xffffffff0e0c0600ULL, + 0xffffffff0e0c0602ULL, 0xffffff0e0c060200ULL, + 0xffffffff0e0c0604ULL, 0xffffff0e0c060400ULL, + 0xffffff0e0c060402ULL, 0xffff0e0c06040200ULL, + 0xffffffffff0e0c08ULL, 0xffffffff0e0c0800ULL, + 0xffffffff0e0c0802ULL, 0xffffff0e0c080200ULL, + 0xffffffff0e0c0804ULL, 0xffffff0e0c080400ULL, + 0xffffff0e0c080402ULL, 0xffff0e0c08040200ULL, + 0xffffffff0e0c0806ULL, 0xffffff0e0c080600ULL, + 0xffffff0e0c080602ULL, 0xffff0e0c08060200ULL, + 0xffffff0e0c080604ULL, 0xffff0e0c08060400ULL, + 0xffff0e0c08060402ULL, 0xff0e0c0806040200ULL, + 0xffffffffff0e0c0aULL, 0xffffffff0e0c0a00ULL, + 0xffffffff0e0c0a02ULL, 0xffffff0e0c0a0200ULL, + 0xffffffff0e0c0a04ULL, 0xffffff0e0c0a0400ULL, + 0xffffff0e0c0a0402ULL, 0xffff0e0c0a040200ULL, + 0xffffffff0e0c0a06ULL, 0xffffff0e0c0a0600ULL, + 0xffffff0e0c0a0602ULL, 0xffff0e0c0a060200ULL, + 0xffffff0e0c0a0604ULL, 0xffff0e0c0a060400ULL, + 0xffff0e0c0a060402ULL, 0xff0e0c0a06040200ULL, + 0xffffffff0e0c0a08ULL, 0xffffff0e0c0a0800ULL, + 0xffffff0e0c0a0802ULL, 0xffff0e0c0a080200ULL, + 0xffffff0e0c0a0804ULL, 0xffff0e0c0a080400ULL, + 0xffff0e0c0a080402ULL, 0xff0e0c0a08040200ULL, + 0xffffff0e0c0a0806ULL, 0xffff0e0c0a080600ULL, + 0xffff0e0c0a080602ULL, 0xff0e0c0a08060200ULL, + 0xffff0e0c0a080604ULL, 0xff0e0c0a08060400ULL, + 0xff0e0c0a08060402ULL, 0x0e0c0a0806040200ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_rej_q[] WC_X64I_UNUSED = { + 0x0d010d010d010d01ULL, 0x0d010d010d010d01ULL, + 0x0d010d010d010d01ULL, 0x0d010d010d010d01ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_rej_ones[] WC_X64I_UNUSED = { + 0x0101010101010101ULL, 0x0101010101010101ULL, + 0x0101010101010101ULL, 0x0101010101010101ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_rej_mask[] WC_X64I_UNUSED = { + 0x0fff0fff0fff0fffULL, 0x0fff0fff0fff0fffULL, + 0x0fff0fff0fff0fffULL, 0x0fff0fff0fff0fffULL, +}; + +XALIGNED(32) static const word64 L_mlkem_rej_shuffle[] WC_X64I_UNUSED = { + 0x0504040302010100ULL, 0x0b0a0a0908070706ULL, + 0x0908080706050504ULL, 0x0f0e0e0d0c0b0b0aULL, +}; + +WC_X64I_TARGET("avx2,bmi,bmi2") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx2(sword16* p, + unsigned int len, const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, r8, rax, r9, r14, rbp, r13, rbx, r10, rcx, r11, r12, + r15 = 0; + __m256i y0, y1, y2, y3, y4 = _mm256_setzero_si256(), + y5 = _mm256_setzero_si256(), y6, y7, y8, y9; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + r8 = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_rej_q, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_rej_ones, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_rej_mask, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_rej_shuffle, 0)); + r9 = (word64)((word64)(size_t)L_mlkem_rej_idx); + r14 = (word64)(0x1111111111111111); + rbp = (word64)(0xe0c0a0806040200); + r13 = (word64)(0x101010101010101); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 24)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 48)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 72)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 120)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 144)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 168)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 216)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 240)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 264)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 312)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + rdx = (word64)(rdx + 0x150); + r8 = (word32)((word32)r8 - 0x150); +L_mlkem_rej_uniform_n_avx2_start_256: + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 24)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + rdx = (word64)(rdx + 0x30); + r8 = (word32)((word32)r8 - 0x30); + if ((sword32)((word32)r8) < (sword32)(0x30)) { + goto L_mlkem_rej_uniform_n_avx2_done_256; + } + if ((sword32)((word32)rsi) >= (sword32)(0x20)) { + goto L_mlkem_rej_uniform_n_avx2_start_256; + } +L_mlkem_rej_uniform_n_avx2_done_256: + if ((sword32)((word32)rsi) < (sword32)(8)) { + goto L_mlkem_rej_uniform_n_avx2_done_128; + } + if ((sword32)((word32)r8) < (sword32)(0xc)) { + goto L_mlkem_rej_uniform_n_avx2_done_128; + } +L_mlkem_rej_uniform_n_avx2_start_128: + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + y0 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y9))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi16(_mm256_castsi256_si128(y0), 4)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xaa)); + y0 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + y2 = _mm256_zextsi128_si256(_mm_cmpgt_epi16(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y0))); + rbx = (word32)((word32)_mm_movemask_epi8(_mm256_castsi256_si128(y2))); + r10 = (word64)(0x5555); + rbx = (word32)((word32)_pext_u32((word32)rbx, (word32)r10)); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rbx * 8)), 0); + y4 = _mm256_zextsi128_si256(_mm_add_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y7))); + y3 = _mm256_zextsi128_si256(_mm_unpacklo_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4))); + y0 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y3))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rcx * 2); + rsi = (word32)((word32)rsi - (word32)rcx); + rdx = (word64)(rdx + 0xc); + r8 = (word32)((word32)r8 - 0xc); + if ((sword32)((word32)r8) < (sword32)(0xc)) { + goto L_mlkem_rej_uniform_n_avx2_done_128; + } + if ((sword32)((word32)rsi) >= (sword32)(8)) { + goto L_mlkem_rej_uniform_n_avx2_start_128; + } +L_mlkem_rej_uniform_n_avx2_done_128: + if (((word32)r8) == (0)) { + goto L_mlkem_rej_uniform_n_avx2_done_64; + } + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx2_done_64; + } + r15 = (word64)(0xfff0fff0fff0fff); + r10 = (word64)(0x2000200020002000); + r11 = (word64)(0xd010d010d010d01); + r12 = (word64)(0x1000100010001000); +L_mlkem_rej_uniform_n_avx2_start_64: + rcx = (word64)(WC_L64(rdx, 0)); + rcx = (word64)((word64)_pdep_u64(rcx, r15)); + if ((sword16)((word16)rcx) >= (sword16)(0xd01)) { + goto L_mlkem_rej_uniform_0_avx2_rej_large_0; + } + WC_S16(rdi, 0) = (word16)((word16)rcx); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx2_done_64; + } +L_mlkem_rej_uniform_0_avx2_rej_large_0: + rcx = (word64)(rcx >> 16); + if ((sword16)((word16)rcx) >= (sword16)(0xd01)) { + goto L_mlkem_rej_uniform_0_avx2_rej_large_1; + } + WC_S16(rdi, 0) = (word16)((word16)rcx); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx2_done_64; + } +L_mlkem_rej_uniform_0_avx2_rej_large_1: + rcx = (word64)(rcx >> 16); + if ((sword16)((word16)rcx) >= (sword16)(0xd01)) { + goto L_mlkem_rej_uniform_0_avx2_rej_large_2; + } + WC_S16(rdi, 0) = (word16)((word16)rcx); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx2_done_64; + } +L_mlkem_rej_uniform_0_avx2_rej_large_2: + rcx = (word64)(rcx >> 16); + if ((sword16)((word16)rcx) >= (sword16)(0xd01)) { + goto L_mlkem_rej_uniform_0_avx2_rej_large_3; + } + WC_S16(rdi, 0) = (word16)((word16)rcx); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx2_done_64; + } +L_mlkem_rej_uniform_0_avx2_rej_large_3: + rdx = (word64)(rdx + 6); + r8 = (word32)((word32)r8 - 6); + if ((sword32)((word32)r8) <= (sword32)(0)) { + goto L_mlkem_rej_uniform_n_avx2_done_64; + } + if ((sword32)((word32)rsi) > (sword32)(0)) { + goto L_mlkem_rej_uniform_n_avx2_start_64; + } +L_mlkem_rej_uniform_n_avx2_done_64: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; + (void)r14; + (void)rbp; + (void)r13; +} + +WC_X64I_TARGET("avx2,bmi,bmi2") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx2(sword16* p, unsigned int len, + const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, r8, rax, r9 = 0, r14 = 0, rbp = 0, r13 = 0, rbx = 0, + r10 = 0, rcx = 0, r11 = 0, r12 = 0, r15 = 0; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), + y4 = _mm256_setzero_si256(), y5 = _mm256_setzero_si256(), + y6 = _mm256_setzero_si256(), y7 = _mm256_setzero_si256(), + y8 = _mm256_setzero_si256(), y9 = _mm256_setzero_si256(); + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + r8 = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx2_done_64; + } + if ((sword32)((word32)rsi) < (sword32)(8)) { + goto L_mlkem_rej_uniform_avx2_done_128; + } + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_rej_q, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_rej_ones, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_rej_mask, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_rej_shuffle, 0)); + r9 = (word64)((word64)(size_t)L_mlkem_rej_idx); + r14 = (word64)(0x1111111111111111); + rbp = (word64)(0xe0c0a0806040200); + r13 = (word64)(0x101010101010101); + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_avx2_done_256; + } + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 24)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + rdx = (word64)(rdx + 0x30); + r8 = (word32)((word32)r8 - 0x30); + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_avx2_done_256; + } + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 24)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + rdx = (word64)(rdx + 0x30); + r8 = (word32)((word32)r8 - 0x30); + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_avx2_done_256; + } +L_mlkem_rej_uniform_avx2_start_256: + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 24)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y9); + y1 = _mm256_shuffle_epi8(y1, y9); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_blend_epi16(y0, y2, 0xaa); + y1 = _mm256_blend_epi16(y1, y3, 0xaa); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_cmpgt_epi16(y6, y0); + y3 = _mm256_cmpgt_epi16(y6, y1); + y2 = _mm256_packs_epi16(y2, y3); + rbx = (word32)((word32)_mm256_movemask_epi8(y2)); + r10 = (word32)((word32)(byte)rbx); + rcx = (word32)((word32)(byte)(rbx >> 8)); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + r11 = (word64)(r11 >> 16); + r12 = (word64)(r12 >> 24); + r11 = (word64)(r11 & 0xff); + r12 = (word64)(r12 & 0xff); + y2 = _mm256_inserti128_si256(y2, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r10 * 8)), 0); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rcx * 8)), 0); + y4 = _mm256_inserti128_si256(y4, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r11 * 8)), 0); + y5 = _mm256_inserti128_si256(y5, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + r12 * 8)), 0); + y2 = _mm256_inserti128_si256(y2, _mm256_castsi256_si128(y4), 1); + y3 = _mm256_inserti128_si256(y3, _mm256_castsi256_si128(y5), 1); + y4 = _mm256_add_epi8(y2, y7); + y5 = _mm256_add_epi8(y3, y7); + y2 = _mm256_unpacklo_epi8(y2, y4); + y3 = _mm256_unpacklo_epi8(y3, y5); + y0 = _mm256_shuffle_epi8(y0, y2); + y1 = _mm256_shuffle_epi8(y1, y3); + r10 = (word64)(rbx); + r11 = (word64)(rbx); + r12 = (word64)(rbx); + rbx = (word64)(rbx & 0xff); + r10 = (word64)(r10 >> 16); + r11 = (word64)(r11 >> 8); + r12 = (word64)(r12 >> 24); + r10 = (word64)(r10 & 0xff); + r11 = (word64)(r11 & 0xff); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + r10 = (word32)(WC_X64I_POPCNT32((word32)r10)); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + r12 = (word32)(WC_X64I_POPCNT32((word32)r12)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + y0 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rdi = (word64)(rdi + r10 * 2); + rsi = (word32)((word32)rsi - (word32)r10); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + rdi = (word64)(rdi + r11 * 2); + rsi = (word32)((word32)rsi - (word32)r11); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y1)); + rdi = (word64)(rdi + r12 * 2); + rsi = (word32)((word32)rsi - (word32)r12); + rdx = (word64)(rdx + 0x30); + r8 = (word32)((word32)r8 - 0x30); + if ((sword32)((word32)r8) < (sword32)(0x30)) { + goto L_mlkem_rej_uniform_avx2_done_256; + } + if ((sword32)((word32)rsi) >= (sword32)(0x20)) { + goto L_mlkem_rej_uniform_avx2_start_256; + } +L_mlkem_rej_uniform_avx2_done_256: + if ((sword32)((word32)rsi) < (sword32)(8)) { + goto L_mlkem_rej_uniform_avx2_done_128; + } + if ((sword32)((word32)r8) < (sword32)(0xc)) { + goto L_mlkem_rej_uniform_avx2_done_128; + } +L_mlkem_rej_uniform_avx2_start_128: + y0 = _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rdx, 0))); + y0 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y9))); + y2 = _mm256_zextsi128_si256(_mm_srli_epi16(_mm256_castsi256_si128(y0), 4)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xaa)); + y0 = _mm256_zextsi128_si256(_mm_and_si128(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y8))); + y2 = _mm256_zextsi128_si256(_mm_cmpgt_epi16(_mm256_castsi256_si128(y6), + _mm256_castsi256_si128(y0))); + rbx = (word32)((word32)_mm_movemask_epi8(_mm256_castsi256_si128(y2))); + r10 = (word64)(0x5555); + rbx = (word32)((word32)_pext_u32((word32)rbx, (word32)r10)); + y3 = _mm256_inserti128_si256(y3, _mm_loadl_epi64((const __m128i*)WC_PR(r9, + rbx * 8)), 0); + y4 = _mm256_zextsi128_si256(_mm_add_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y7))); + y3 = _mm256_zextsi128_si256(_mm_unpacklo_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4))); + y0 = _mm256_zextsi128_si256(_mm_shuffle_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y3))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + rcx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rcx * 2); + rsi = (word32)((word32)rsi - (word32)rcx); + rdx = (word64)(rdx + 0xc); + r8 = (word32)((word32)r8 - 0xc); + if ((sword32)((word32)r8) < (sword32)(0xc)) { + goto L_mlkem_rej_uniform_avx2_done_128; + } + if ((sword32)((word32)rsi) >= (sword32)(8)) { + goto L_mlkem_rej_uniform_avx2_start_128; + } +L_mlkem_rej_uniform_avx2_done_128: + if (((word32)r8) == (0)) { + goto L_mlkem_rej_uniform_avx2_done_64; + } + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx2_done_64; + } + r15 = (word64)(0xfff0fff0fff0fff); + r10 = (word64)(0x2000200020002000); + r11 = (word64)(0xd010d010d010d01); + r12 = (word64)(0x1000100010001000); +L_mlkem_rej_uniform_avx2_start_64: + rcx = (word64)(WC_L64(rdx, 0)); + rcx = (word64)((word64)_pdep_u64(rcx, r15)); + if ((sword16)((word16)rcx) >= (sword16)(0xd01)) { + goto L_mlkem_rej_uniform_avx2_rej_large_0; + } + WC_S16(rdi, 0) = (word16)((word16)rcx); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx2_done_64; + } +L_mlkem_rej_uniform_avx2_rej_large_0: + rcx = (word64)(rcx >> 16); + if ((sword16)((word16)rcx) >= (sword16)(0xd01)) { + goto L_mlkem_rej_uniform_avx2_rej_large_1; + } + WC_S16(rdi, 0) = (word16)((word16)rcx); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx2_done_64; + } +L_mlkem_rej_uniform_avx2_rej_large_1: + rcx = (word64)(rcx >> 16); + if ((sword16)((word16)rcx) >= (sword16)(0xd01)) { + goto L_mlkem_rej_uniform_avx2_rej_large_2; + } + WC_S16(rdi, 0) = (word16)((word16)rcx); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx2_done_64; + } +L_mlkem_rej_uniform_avx2_rej_large_2: + rcx = (word64)(rcx >> 16); + if ((sword16)((word16)rcx) >= (sword16)(0xd01)) { + goto L_mlkem_rej_uniform_avx2_rej_large_3; + } + WC_S16(rdi, 0) = (word16)((word16)rcx); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx2_done_64; + } +L_mlkem_rej_uniform_avx2_rej_large_3: + rdx = (word64)(rdx + 6); + r8 = (word32)((word32)r8 - 6); + if ((sword32)((word32)r8) <= (sword32)(0)) { + goto L_mlkem_rej_uniform_avx2_done_64; + } + if ((sword32)((word32)rsi) > (sword32)(0)) { + goto L_mlkem_rej_uniform_avx2_start_64; + } +L_mlkem_rej_uniform_avx2_done_64: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; + (void)r14; + (void)rbp; + (void)r13; +} + +XALIGNED(32) static const word64 L_mlkem_mask_249[] WC_X64I_UNUSED = { + 0x0024924900249249ULL, 0x0024924900249249ULL, + 0x0024924900249249ULL, 0x0024924900249249ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_mask_6db[] WC_X64I_UNUSED = { + 0x006db6db006db6dbULL, 0x006db6db006db6dbULL, + 0x006db6db006db6dbULL, 0x006db6db006db6dbULL, +}; + +XALIGNED(32) static const word64 L_mlkem_mask_07[] WC_X64I_UNUSED = { + 0x0000000700000007ULL, 0x0000000700000007ULL, + 0x0000000700000007ULL, 0x0000000700000007ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_mask_70[] WC_X64I_UNUSED = { + 0x0007000000070000ULL, 0x0007000000070000ULL, + 0x0007000000070000ULL, 0x0007000000070000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_mask_3[] WC_X64I_UNUSED = { + 0x0003000300030003ULL, 0x0003000300030003ULL, + 0x0003000300030003ULL, 0x0003000300030003ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_shuff[] WC_X64I_UNUSED = { + 0xff050403ff020100ULL, 0xff0b0a09ff080706ULL, + 0xff090807ff060504ULL, 0xff0f0e0dff0c0b0aULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_cbd_eta3_avx2(sword16* p, const byte* r) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_mask_249, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_mask_6db, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_mask_07, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_mask_70, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_mask_3, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_shuff, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 24)); + y0 = _mm256_permute4x64_epi64(y0, 0x94); + y1 = _mm256_permute4x64_epi64(y1, 0x94); + y0 = _mm256_shuffle_epi8(y0, y13); + y1 = _mm256_shuffle_epi8(y1, y13); + y2 = _mm256_srli_epi32(y0, 1); + y3 = _mm256_srli_epi32(y1, 1); + y4 = _mm256_srli_epi32(y0, 2); + y5 = _mm256_srli_epi32(y1, 2); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y4 = _mm256_and_si256(y4, y8); + y5 = _mm256_and_si256(y5, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_srli_epi32(y0, 3); + y3 = _mm256_srli_epi32(y1, 3); + y0 = _mm256_add_epi32(y0, y9); + y1 = _mm256_add_epi32(y1, y9); + y0 = _mm256_sub_epi32(y0, y2); + y1 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_slli_epi32(y0, 10); + y3 = _mm256_slli_epi32(y1, 10); + y4 = _mm256_srli_epi32(y0, 12); + y5 = _mm256_srli_epi32(y1, 12); + y6 = _mm256_srli_epi32(y0, 2); + y7 = _mm256_srli_epi32(y1, 2); + y0 = _mm256_and_si256(y0, y10); + y1 = _mm256_and_si256(y1, y10); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y4 = _mm256_and_si256(y4, y10); + y5 = _mm256_and_si256(y5, y10); + y6 = _mm256_and_si256(y6, y11); + y7 = _mm256_and_si256(y7, y11); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_add_epi16(y4, y6); + y3 = _mm256_add_epi16(y5, y7); + y0 = _mm256_sub_epi16(y0, y12); + y1 = _mm256_sub_epi16(y1, y12); + y2 = _mm256_sub_epi16(y2, y12); + y3 = _mm256_sub_epi16(y3, y12); + y4 = _mm256_unpacklo_epi32(y0, y2); + y5 = _mm256_unpacklo_epi32(y1, y3); + y6 = _mm256_unpackhi_epi32(y0, y2); + y7 = _mm256_unpackhi_epi32(y1, y3); + y0 = _mm256_permute2x128_si256(y4, y6, 0x20); + y1 = _mm256_permute2x128_si256(y5, y7, 0x20); + y2 = _mm256_permute2x128_si256(y4, y6, 0x31); + y3 = _mm256_permute2x128_si256(y5, y7, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 48)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 72)); + y0 = _mm256_permute4x64_epi64(y0, 0x94); + y1 = _mm256_permute4x64_epi64(y1, 0x94); + y0 = _mm256_shuffle_epi8(y0, y13); + y1 = _mm256_shuffle_epi8(y1, y13); + y2 = _mm256_srli_epi32(y0, 1); + y3 = _mm256_srli_epi32(y1, 1); + y4 = _mm256_srli_epi32(y0, 2); + y5 = _mm256_srli_epi32(y1, 2); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y4 = _mm256_and_si256(y4, y8); + y5 = _mm256_and_si256(y5, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_srli_epi32(y0, 3); + y3 = _mm256_srli_epi32(y1, 3); + y0 = _mm256_add_epi32(y0, y9); + y1 = _mm256_add_epi32(y1, y9); + y0 = _mm256_sub_epi32(y0, y2); + y1 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_slli_epi32(y0, 10); + y3 = _mm256_slli_epi32(y1, 10); + y4 = _mm256_srli_epi32(y0, 12); + y5 = _mm256_srli_epi32(y1, 12); + y6 = _mm256_srli_epi32(y0, 2); + y7 = _mm256_srli_epi32(y1, 2); + y0 = _mm256_and_si256(y0, y10); + y1 = _mm256_and_si256(y1, y10); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y4 = _mm256_and_si256(y4, y10); + y5 = _mm256_and_si256(y5, y10); + y6 = _mm256_and_si256(y6, y11); + y7 = _mm256_and_si256(y7, y11); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_add_epi16(y4, y6); + y3 = _mm256_add_epi16(y5, y7); + y0 = _mm256_sub_epi16(y0, y12); + y1 = _mm256_sub_epi16(y1, y12); + y2 = _mm256_sub_epi16(y2, y12); + y3 = _mm256_sub_epi16(y3, y12); + y4 = _mm256_unpacklo_epi32(y0, y2); + y5 = _mm256_unpacklo_epi32(y1, y3); + y6 = _mm256_unpackhi_epi32(y0, y2); + y7 = _mm256_unpackhi_epi32(y1, y3); + y0 = _mm256_permute2x128_si256(y4, y6, 0x20); + y1 = _mm256_permute2x128_si256(y5, y7, 0x20); + y2 = _mm256_permute2x128_si256(y4, y6, 0x31); + y3 = _mm256_permute2x128_si256(y5, y7, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 120)); + y0 = _mm256_permute4x64_epi64(y0, 0x94); + y1 = _mm256_permute4x64_epi64(y1, 0x94); + y0 = _mm256_shuffle_epi8(y0, y13); + y1 = _mm256_shuffle_epi8(y1, y13); + y2 = _mm256_srli_epi32(y0, 1); + y3 = _mm256_srli_epi32(y1, 1); + y4 = _mm256_srli_epi32(y0, 2); + y5 = _mm256_srli_epi32(y1, 2); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y4 = _mm256_and_si256(y4, y8); + y5 = _mm256_and_si256(y5, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_srli_epi32(y0, 3); + y3 = _mm256_srli_epi32(y1, 3); + y0 = _mm256_add_epi32(y0, y9); + y1 = _mm256_add_epi32(y1, y9); + y0 = _mm256_sub_epi32(y0, y2); + y1 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_slli_epi32(y0, 10); + y3 = _mm256_slli_epi32(y1, 10); + y4 = _mm256_srli_epi32(y0, 12); + y5 = _mm256_srli_epi32(y1, 12); + y6 = _mm256_srli_epi32(y0, 2); + y7 = _mm256_srli_epi32(y1, 2); + y0 = _mm256_and_si256(y0, y10); + y1 = _mm256_and_si256(y1, y10); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y4 = _mm256_and_si256(y4, y10); + y5 = _mm256_and_si256(y5, y10); + y6 = _mm256_and_si256(y6, y11); + y7 = _mm256_and_si256(y7, y11); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_add_epi16(y4, y6); + y3 = _mm256_add_epi16(y5, y7); + y0 = _mm256_sub_epi16(y0, y12); + y1 = _mm256_sub_epi16(y1, y12); + y2 = _mm256_sub_epi16(y2, y12); + y3 = _mm256_sub_epi16(y3, y12); + y4 = _mm256_unpacklo_epi32(y0, y2); + y5 = _mm256_unpacklo_epi32(y1, y3); + y6 = _mm256_unpackhi_epi32(y0, y2); + y7 = _mm256_unpackhi_epi32(y1, y3); + y0 = _mm256_permute2x128_si256(y4, y6, 0x20); + y1 = _mm256_permute2x128_si256(y5, y7, 0x20); + y2 = _mm256_permute2x128_si256(y4, y6, 0x31); + y3 = _mm256_permute2x128_si256(y5, y7, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 144)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 168)); + y0 = _mm256_permute4x64_epi64(y0, 0x94); + y1 = _mm256_permute4x64_epi64(y1, 0x94); + y0 = _mm256_shuffle_epi8(y0, y13); + y1 = _mm256_shuffle_epi8(y1, y13); + y2 = _mm256_srli_epi32(y0, 1); + y3 = _mm256_srli_epi32(y1, 1); + y4 = _mm256_srli_epi32(y0, 2); + y5 = _mm256_srli_epi32(y1, 2); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y4 = _mm256_and_si256(y4, y8); + y5 = _mm256_and_si256(y5, y8); + y0 = _mm256_add_epi32(y0, y2); + y1 = _mm256_add_epi32(y1, y3); + y0 = _mm256_add_epi32(y0, y4); + y1 = _mm256_add_epi32(y1, y5); + y2 = _mm256_srli_epi32(y0, 3); + y3 = _mm256_srli_epi32(y1, 3); + y0 = _mm256_add_epi32(y0, y9); + y1 = _mm256_add_epi32(y1, y9); + y0 = _mm256_sub_epi32(y0, y2); + y1 = _mm256_sub_epi32(y1, y3); + y2 = _mm256_slli_epi32(y0, 10); + y3 = _mm256_slli_epi32(y1, 10); + y4 = _mm256_srli_epi32(y0, 12); + y5 = _mm256_srli_epi32(y1, 12); + y6 = _mm256_srli_epi32(y0, 2); + y7 = _mm256_srli_epi32(y1, 2); + y0 = _mm256_and_si256(y0, y10); + y1 = _mm256_and_si256(y1, y10); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y4 = _mm256_and_si256(y4, y10); + y5 = _mm256_and_si256(y5, y10); + y6 = _mm256_and_si256(y6, y11); + y7 = _mm256_and_si256(y7, y11); + y0 = _mm256_add_epi16(y0, y2); + y1 = _mm256_add_epi16(y1, y3); + y2 = _mm256_add_epi16(y4, y6); + y3 = _mm256_add_epi16(y5, y7); + y0 = _mm256_sub_epi16(y0, y12); + y1 = _mm256_sub_epi16(y1, y12); + y2 = _mm256_sub_epi16(y2, y12); + y3 = _mm256_sub_epi16(y3, y12); + y4 = _mm256_unpacklo_epi32(y0, y2); + y5 = _mm256_unpacklo_epi32(y1, y3); + y6 = _mm256_unpackhi_epi32(y0, y2); + y7 = _mm256_unpackhi_epi32(y1, y3); + y0 = _mm256_permute2x128_si256(y4, y6, 0x20); + y1 = _mm256_permute2x128_si256(y5, y7, 0x20); + y2 = _mm256_permute2x128_si256(y4, y6, 0x31); + y3 = _mm256_permute2x128_si256(y5, y7, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); +} + +XALIGNED(32) static const word64 L_mlkem_mask_55[] WC_X64I_UNUSED = { + 0x5555555555555555ULL, 0x5555555555555555ULL, + 0x5555555555555555ULL, 0x5555555555555555ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_mask_33[] WC_X64I_UNUSED = { + 0x3333333333333333ULL, 0x3333333333333333ULL, + 0x3333333333333333ULL, 0x3333333333333333ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_mask_03[] WC_X64I_UNUSED = { + 0x0303030303030303ULL, 0x0303030303030303ULL, + 0x0303030303030303ULL, 0x0303030303030303ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_mask_0f[] WC_X64I_UNUSED = { + 0x0f0f0f0f0f0f0f0fULL, 0x0f0f0f0f0f0f0f0fULL, + 0x0f0f0f0f0f0f0f0fULL, 0x0f0f0f0f0f0f0f0fULL, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_cbd_eta2_avx2(sword16* p, const byte* r) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_mask_55, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_mask_33, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_mask_03, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_mask_0f, 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_srli_epi16(y0, 1); + y3 = _mm256_srli_epi16(y1, 1); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi8(y0, y2); + y1 = _mm256_add_epi8(y1, y3); + y2 = _mm256_srli_epi16(y0, 2); + y3 = _mm256_srli_epi16(y1, 2); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y2 = _mm256_and_si256(y2, y9); + y3 = _mm256_and_si256(y3, y9); + y0 = _mm256_add_epi8(y0, y9); + y1 = _mm256_add_epi8(y1, y9); + y0 = _mm256_sub_epi8(y0, y2); + y1 = _mm256_sub_epi8(y1, y3); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_and_si256(y0, y11); + y1 = _mm256_and_si256(y1, y11); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y0 = _mm256_sub_epi8(y0, y10); + y1 = _mm256_sub_epi8(y1, y10); + y2 = _mm256_sub_epi8(y2, y10); + y3 = _mm256_sub_epi8(y3, y10); + y4 = _mm256_unpacklo_epi8(y0, y2); + y5 = _mm256_unpacklo_epi8(y1, y3); + y6 = _mm256_unpackhi_epi8(y0, y2); + y7 = _mm256_unpackhi_epi8(y1, y3); + y0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y4)); + y1 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y5)); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y3 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y2 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y2)); + y3 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y3)); + y4 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y6)); + y5 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y7)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y6, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y7, 1)); + y6 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y6)); + y7 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y2 = _mm256_srli_epi16(y0, 1); + y3 = _mm256_srli_epi16(y1, 1); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_add_epi8(y0, y2); + y1 = _mm256_add_epi8(y1, y3); + y2 = _mm256_srli_epi16(y0, 2); + y3 = _mm256_srli_epi16(y1, 2); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y2 = _mm256_and_si256(y2, y9); + y3 = _mm256_and_si256(y3, y9); + y0 = _mm256_add_epi8(y0, y9); + y1 = _mm256_add_epi8(y1, y9); + y0 = _mm256_sub_epi8(y0, y2); + y1 = _mm256_sub_epi8(y1, y3); + y2 = _mm256_srli_epi16(y0, 4); + y3 = _mm256_srli_epi16(y1, 4); + y0 = _mm256_and_si256(y0, y11); + y1 = _mm256_and_si256(y1, y11); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y0 = _mm256_sub_epi8(y0, y10); + y1 = _mm256_sub_epi8(y1, y10); + y2 = _mm256_sub_epi8(y2, y10); + y3 = _mm256_sub_epi8(y3, y10); + y4 = _mm256_unpacklo_epi8(y0, y2); + y5 = _mm256_unpacklo_epi8(y1, y3); + y6 = _mm256_unpackhi_epi8(y0, y2); + y7 = _mm256_unpackhi_epi8(y1, y3); + y0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y4)); + y1 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y5)); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y4, 1)); + y3 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y5, 1)); + y2 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y2)); + y3 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y3)); + y4 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y6)); + y5 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y7)); + y6 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y6, 1)); + y7 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y7, 1)); + y6 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y6)); + y7 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); +} + +XALIGNED(16) static const word16 L_mlkem_compress_10_avx2_mask[] + WC_X64I_UNUSED = { + 0x03ff, 0x03ff, 0x03ff, 0x03ff, 0x03ff, 0x03ff, 0x03ff, 0x03ff, + 0x03ff, 0x03ff, 0x03ff, 0x03ff, 0x03ff, 0x03ff, 0x03ff, 0x03ff, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_10_avx2_shift[] + WC_X64I_UNUSED = { + 0x0400000104000001ULL, 0x0400000104000001ULL, + 0x0400000104000001ULL, 0x0400000104000001ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_10_avx2_shlv[] + WC_X64I_UNUSED = { + 0x000000000000000cULL, 0x000000000000000cULL, + 0x000000000000000cULL, 0x000000000000000cULL, +}; + +XALIGNED(16) static const byte L_mlkem_compress_10_avx2_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x09, 0x0a, 0x0b, 0x0c, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_10_avx2_v[] WC_X64I_UNUSED = { + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_10_avx2_offset[] + WC_X64I_UNUSED = { + 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, + 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_10_avx2_shift12[] + WC_X64I_UNUSED = { + 0x1000, 0x1000, 0x1000, 0x1000, 0x1000, 0x1000, 0x1000, 0x1000, + 0x1000, 0x1000, 0x1000, 0x1000, 0x1000, 0x1000, 0x1000, 0x1000, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_compress_10_avx2(byte* r, const sword16* p, int n) +{ + word64 rdi, rsi, rdx; + __m256i y0, y1 = _mm256_setzero_si256(), y2 = _mm256_setzero_si256(), + y3 = _mm256_setzero_si256(), y4 = _mm256_setzero_si256(), + y5 = _mm256_setzero_si256(), y6, y7, y8, y9, y10, y11, y12, y13; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + rdx = (word64)(word32)n; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_10_avx2_mask, + 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_10_avx2_shift, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_10_avx2_shlv, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_10_avx2_shuf, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_10_avx2_v, + 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_10_avx2_offset, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_10_avx2_shift12, 0)); + y7 = _mm256_slli_epi16(y6, 3); +L_mlkem_compress_10_avx2_start: + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_mullo_epi16(y0, y7); + y4 = _mm256_mullo_epi16(y1, y7); + y3 = _mm256_add_epi16(y0, y12); + y5 = _mm256_add_epi16(y1, y12); + y0 = _mm256_slli_epi16(y0, 3); + y1 = _mm256_slli_epi16(y1, 3); + y0 = _mm256_mulhi_epu16(y0, y6); + y1 = _mm256_mulhi_epu16(y1, y6); + y3 = _mm256_sub_epi16(y2, y3); + y5 = _mm256_sub_epi16(y4, y5); + y2 = _mm256_andnot_si256(y2, y3); + y4 = _mm256_andnot_si256(y4, y5); + y2 = _mm256_srli_epi16(y2, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y2); + y1 = _mm256_sub_epi16(y1, y4); + y0 = _mm256_mulhrs_epi16(y0, y13); + y1 = _mm256_mulhrs_epi16(y1, y13); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y0 = _mm256_madd_epi16(y0, y8); + y1 = _mm256_madd_epi16(y1, y8); + y0 = _mm256_sllv_epi32(y0, y10); + y1 = _mm256_sllv_epi32(y1, y10); + y0 = _mm256_srli_epi64(y0, 12); + y1 = _mm256_srli_epi64(y1, 12); + y0 = _mm256_shuffle_epi8(y0, y11); + y1 = _mm256_shuffle_epi8(y1, y11); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xe0)); + y1 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4), 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 20), _mm256_castsi256_si128(y1)); + WC_S32(rdi, 16) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y2)); + WC_S32(rdi, 36) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y2 = _mm256_mullo_epi16(y0, y7); + y4 = _mm256_mullo_epi16(y1, y7); + y3 = _mm256_add_epi16(y0, y12); + y5 = _mm256_add_epi16(y1, y12); + y0 = _mm256_slli_epi16(y0, 3); + y1 = _mm256_slli_epi16(y1, 3); + y0 = _mm256_mulhi_epu16(y0, y6); + y1 = _mm256_mulhi_epu16(y1, y6); + y3 = _mm256_sub_epi16(y2, y3); + y5 = _mm256_sub_epi16(y4, y5); + y2 = _mm256_andnot_si256(y2, y3); + y4 = _mm256_andnot_si256(y4, y5); + y2 = _mm256_srli_epi16(y2, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y2); + y1 = _mm256_sub_epi16(y1, y4); + y0 = _mm256_mulhrs_epi16(y0, y13); + y1 = _mm256_mulhrs_epi16(y1, y13); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y0 = _mm256_madd_epi16(y0, y8); + y1 = _mm256_madd_epi16(y1, y8); + y0 = _mm256_sllv_epi32(y0, y10); + y1 = _mm256_sllv_epi32(y1, y10); + y0 = _mm256_srli_epi64(y0, 12); + y1 = _mm256_srli_epi64(y1, 12); + y0 = _mm256_shuffle_epi8(y0, y11); + y1 = _mm256_shuffle_epi8(y1, y11); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xe0)); + y1 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4), 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 40), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 60), _mm256_castsi256_si128(y1)); + WC_S32(rdi, 56) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y2)); + WC_S32(rdi, 76) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y2 = _mm256_mullo_epi16(y0, y7); + y4 = _mm256_mullo_epi16(y1, y7); + y3 = _mm256_add_epi16(y0, y12); + y5 = _mm256_add_epi16(y1, y12); + y0 = _mm256_slli_epi16(y0, 3); + y1 = _mm256_slli_epi16(y1, 3); + y0 = _mm256_mulhi_epu16(y0, y6); + y1 = _mm256_mulhi_epu16(y1, y6); + y3 = _mm256_sub_epi16(y2, y3); + y5 = _mm256_sub_epi16(y4, y5); + y2 = _mm256_andnot_si256(y2, y3); + y4 = _mm256_andnot_si256(y4, y5); + y2 = _mm256_srli_epi16(y2, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y2); + y1 = _mm256_sub_epi16(y1, y4); + y0 = _mm256_mulhrs_epi16(y0, y13); + y1 = _mm256_mulhrs_epi16(y1, y13); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y0 = _mm256_madd_epi16(y0, y8); + y1 = _mm256_madd_epi16(y1, y8); + y0 = _mm256_sllv_epi32(y0, y10); + y1 = _mm256_sllv_epi32(y1, y10); + y0 = _mm256_srli_epi64(y0, 12); + y1 = _mm256_srli_epi64(y1, 12); + y0 = _mm256_shuffle_epi8(y0, y11); + y1 = _mm256_shuffle_epi8(y1, y11); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xe0)); + y1 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4), 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 100), _mm256_castsi256_si128(y1)); + WC_S32(rdi, 96) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y2)); + WC_S32(rdi, 116) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y2 = _mm256_mullo_epi16(y0, y7); + y4 = _mm256_mullo_epi16(y1, y7); + y3 = _mm256_add_epi16(y0, y12); + y5 = _mm256_add_epi16(y1, y12); + y0 = _mm256_slli_epi16(y0, 3); + y1 = _mm256_slli_epi16(y1, 3); + y0 = _mm256_mulhi_epu16(y0, y6); + y1 = _mm256_mulhi_epu16(y1, y6); + y3 = _mm256_sub_epi16(y2, y3); + y5 = _mm256_sub_epi16(y4, y5); + y2 = _mm256_andnot_si256(y2, y3); + y4 = _mm256_andnot_si256(y4, y5); + y2 = _mm256_srli_epi16(y2, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y2); + y1 = _mm256_sub_epi16(y1, y4); + y0 = _mm256_mulhrs_epi16(y0, y13); + y1 = _mm256_mulhrs_epi16(y1, y13); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y0 = _mm256_madd_epi16(y0, y8); + y1 = _mm256_madd_epi16(y1, y8); + y0 = _mm256_sllv_epi32(y0, y10); + y1 = _mm256_sllv_epi32(y1, y10); + y0 = _mm256_srli_epi64(y0, 12); + y1 = _mm256_srli_epi64(y1, 12); + y0 = _mm256_shuffle_epi8(y0, y11); + y1 = _mm256_shuffle_epi8(y1, y11); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xe0)); + y1 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4), 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 120), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 140), _mm256_castsi256_si128(y1)); + WC_S32(rdi, 136) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y2)); + WC_S32(rdi, 156) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y2 = _mm256_mullo_epi16(y0, y7); + y4 = _mm256_mullo_epi16(y1, y7); + y3 = _mm256_add_epi16(y0, y12); + y5 = _mm256_add_epi16(y1, y12); + y0 = _mm256_slli_epi16(y0, 3); + y1 = _mm256_slli_epi16(y1, 3); + y0 = _mm256_mulhi_epu16(y0, y6); + y1 = _mm256_mulhi_epu16(y1, y6); + y3 = _mm256_sub_epi16(y2, y3); + y5 = _mm256_sub_epi16(y4, y5); + y2 = _mm256_andnot_si256(y2, y3); + y4 = _mm256_andnot_si256(y4, y5); + y2 = _mm256_srli_epi16(y2, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y2); + y1 = _mm256_sub_epi16(y1, y4); + y0 = _mm256_mulhrs_epi16(y0, y13); + y1 = _mm256_mulhrs_epi16(y1, y13); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y0 = _mm256_madd_epi16(y0, y8); + y1 = _mm256_madd_epi16(y1, y8); + y0 = _mm256_sllv_epi32(y0, y10); + y1 = _mm256_sllv_epi32(y1, y10); + y0 = _mm256_srli_epi64(y0, 12); + y1 = _mm256_srli_epi64(y1, 12); + y0 = _mm256_shuffle_epi8(y0, y11); + y1 = _mm256_shuffle_epi8(y1, y11); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xe0)); + y1 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4), 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 160), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 180), _mm256_castsi256_si128(y1)); + WC_S32(rdi, 176) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y2)); + WC_S32(rdi, 196) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y2 = _mm256_mullo_epi16(y0, y7); + y4 = _mm256_mullo_epi16(y1, y7); + y3 = _mm256_add_epi16(y0, y12); + y5 = _mm256_add_epi16(y1, y12); + y0 = _mm256_slli_epi16(y0, 3); + y1 = _mm256_slli_epi16(y1, 3); + y0 = _mm256_mulhi_epu16(y0, y6); + y1 = _mm256_mulhi_epu16(y1, y6); + y3 = _mm256_sub_epi16(y2, y3); + y5 = _mm256_sub_epi16(y4, y5); + y2 = _mm256_andnot_si256(y2, y3); + y4 = _mm256_andnot_si256(y4, y5); + y2 = _mm256_srli_epi16(y2, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y2); + y1 = _mm256_sub_epi16(y1, y4); + y0 = _mm256_mulhrs_epi16(y0, y13); + y1 = _mm256_mulhrs_epi16(y1, y13); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y0 = _mm256_madd_epi16(y0, y8); + y1 = _mm256_madd_epi16(y1, y8); + y0 = _mm256_sllv_epi32(y0, y10); + y1 = _mm256_sllv_epi32(y1, y10); + y0 = _mm256_srli_epi64(y0, 12); + y1 = _mm256_srli_epi64(y1, 12); + y0 = _mm256_shuffle_epi8(y0, y11); + y1 = _mm256_shuffle_epi8(y1, y11); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xe0)); + y1 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4), 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 200), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 220), _mm256_castsi256_si128(y1)); + WC_S32(rdi, 216) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y2)); + WC_S32(rdi, 236) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y2 = _mm256_mullo_epi16(y0, y7); + y4 = _mm256_mullo_epi16(y1, y7); + y3 = _mm256_add_epi16(y0, y12); + y5 = _mm256_add_epi16(y1, y12); + y0 = _mm256_slli_epi16(y0, 3); + y1 = _mm256_slli_epi16(y1, 3); + y0 = _mm256_mulhi_epu16(y0, y6); + y1 = _mm256_mulhi_epu16(y1, y6); + y3 = _mm256_sub_epi16(y2, y3); + y5 = _mm256_sub_epi16(y4, y5); + y2 = _mm256_andnot_si256(y2, y3); + y4 = _mm256_andnot_si256(y4, y5); + y2 = _mm256_srli_epi16(y2, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y2); + y1 = _mm256_sub_epi16(y1, y4); + y0 = _mm256_mulhrs_epi16(y0, y13); + y1 = _mm256_mulhrs_epi16(y1, y13); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y0 = _mm256_madd_epi16(y0, y8); + y1 = _mm256_madd_epi16(y1, y8); + y0 = _mm256_sllv_epi32(y0, y10); + y1 = _mm256_sllv_epi32(y1, y10); + y0 = _mm256_srli_epi64(y0, 12); + y1 = _mm256_srli_epi64(y1, 12); + y0 = _mm256_shuffle_epi8(y0, y11); + y1 = _mm256_shuffle_epi8(y1, y11); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xe0)); + y1 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4), 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 240), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 260), _mm256_castsi256_si128(y1)); + WC_S32(rdi, 256) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y2)); + WC_S32(rdi, 276) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y2 = _mm256_mullo_epi16(y0, y7); + y4 = _mm256_mullo_epi16(y1, y7); + y3 = _mm256_add_epi16(y0, y12); + y5 = _mm256_add_epi16(y1, y12); + y0 = _mm256_slli_epi16(y0, 3); + y1 = _mm256_slli_epi16(y1, 3); + y0 = _mm256_mulhi_epu16(y0, y6); + y1 = _mm256_mulhi_epu16(y1, y6); + y3 = _mm256_sub_epi16(y2, y3); + y5 = _mm256_sub_epi16(y4, y5); + y2 = _mm256_andnot_si256(y2, y3); + y4 = _mm256_andnot_si256(y4, y5); + y2 = _mm256_srli_epi16(y2, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y2); + y1 = _mm256_sub_epi16(y1, y4); + y0 = _mm256_mulhrs_epi16(y0, y13); + y1 = _mm256_mulhrs_epi16(y1, y13); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y0 = _mm256_madd_epi16(y0, y8); + y1 = _mm256_madd_epi16(y1, y8); + y0 = _mm256_sllv_epi32(y0, y10); + y1 = _mm256_sllv_epi32(y1, y10); + y0 = _mm256_srli_epi64(y0, 12); + y1 = _mm256_srli_epi64(y1, 12); + y0 = _mm256_shuffle_epi8(y0, y11); + y1 = _mm256_shuffle_epi8(y1, y11); + y2 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y1, 1)); + y0 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y2), 0xe0)); + y1 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y1), + _mm256_castsi256_si128(y4), 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 280), _mm256_castsi256_si128(y0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 300), _mm256_castsi256_si128(y1)); + WC_S32(rdi, 296) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y2)); + WC_S32(rdi, 316) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y4)); + rdi = (word64)(rdi + 0x140); + rsi = (word64)(rsi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_compress_10_avx2_start; + } +} + +XALIGNED(16) static const word32 L_mlkem_decompress_10_avx2_mask[] + WC_X64I_UNUSED = { + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, +}; + +XALIGNED(32) static const word64 L_mlkem_decompress_10_avx2_sllv[] + WC_X64I_UNUSED = { + 0x0000000000000004ULL, 0x0000000000000004ULL, + 0x0000000000000004ULL, 0x0000000000000004ULL, +}; + +XALIGNED(16) static const word32 L_mlkem_decompress_10_avx2_q[] + WC_X64I_UNUSED = { + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_10_avx2_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, + 0x05, 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, + 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, + 0x07, 0x08, 0x08, 0x09, 0x09, 0x0a, 0x0a, 0x0b, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_decompress_10_avx2(sword16* p, const byte* r, int n) +{ + word64 rdi, rsi, rdx; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), y4, y5, + y6, y7; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + rdx = (word64)(word32)n; + + y4 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_10_avx2_mask, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_decompress_10_avx2_q, + 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_10_avx2_shuf, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_10_avx2_sllv, 0)); +L_mlkem_decompress_10_avx2_start: + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 20)), 0x94); + y2 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 40)), 0x94); + y3 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 60)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_shuffle_epi8(y3, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_srli_epi16(y0, 1); + y1 = _mm256_srli_epi16(y1, 1); + y2 = _mm256_srli_epi16(y2, 1); + y3 = _mm256_srli_epi16(y3, 1); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y5); + y1 = _mm256_mulhrs_epi16(y1, y5); + y2 = _mm256_mulhrs_epi16(y2, y5); + y3 = _mm256_mulhrs_epi16(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 80)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 100)), 0x94); + y2 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 120)), 0x94); + y3 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 140)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_shuffle_epi8(y3, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_srli_epi16(y0, 1); + y1 = _mm256_srli_epi16(y1, 1); + y2 = _mm256_srli_epi16(y2, 1); + y3 = _mm256_srli_epi16(y3, 1); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y5); + y1 = _mm256_mulhrs_epi16(y1, y5); + y2 = _mm256_mulhrs_epi16(y2, y5); + y3 = _mm256_mulhrs_epi16(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y3); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 180)), 0x94); + y2 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 200)), 0x94); + y3 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 220)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_shuffle_epi8(y3, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_srli_epi16(y0, 1); + y1 = _mm256_srli_epi16(y1, 1); + y2 = _mm256_srli_epi16(y2, 1); + y3 = _mm256_srli_epi16(y3, 1); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y5); + y1 = _mm256_mulhrs_epi16(y1, y5); + y2 = _mm256_mulhrs_epi16(y2, y5); + y3 = _mm256_mulhrs_epi16(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 240)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 260)), 0x94); + y2 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 280)), 0x94); + y3 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 300)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_shuffle_epi8(y3, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y1 = _mm256_sllv_epi32(y1, y7); + y2 = _mm256_sllv_epi32(y2, y7); + y3 = _mm256_sllv_epi32(y3, y7); + y0 = _mm256_srli_epi16(y0, 1); + y1 = _mm256_srli_epi16(y1, 1); + y2 = _mm256_srli_epi16(y2, 1); + y3 = _mm256_srli_epi16(y3, 1); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y5); + y1 = _mm256_mulhrs_epi16(y1, y5); + y2 = _mm256_mulhrs_epi16(y2, y5); + y3 = _mm256_mulhrs_epi16(y3, y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); + rsi = (word64)(rsi + 0x140); + rdi = (word64)(rdi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_decompress_10_avx2_start; + } +} + +XALIGNED(16) static const word16 L_mlkem_compress_11_avx2_v[] WC_X64I_UNUSED = { + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_11_avx2_off[] + WC_X64I_UNUSED = { + 0x0024, 0x0024, 0x0024, 0x0024, 0x0024, 0x0024, 0x0024, 0x0024, + 0x0024, 0x0024, 0x0024, 0x0024, 0x0024, 0x0024, 0x0024, 0x0024, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_11_avx2_shift13[] + WC_X64I_UNUSED = { + 0x2000, 0x2000, 0x2000, 0x2000, 0x2000, 0x2000, 0x2000, 0x2000, + 0x2000, 0x2000, 0x2000, 0x2000, 0x2000, 0x2000, 0x2000, 0x2000, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_11_avx2_mask[] + WC_X64I_UNUSED = { + 0x07ff, 0x07ff, 0x07ff, 0x07ff, 0x07ff, 0x07ff, 0x07ff, 0x07ff, + 0x07ff, 0x07ff, 0x07ff, 0x07ff, 0x07ff, 0x07ff, 0x07ff, 0x07ff, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_11_avx2_shift[] + WC_X64I_UNUSED = { + 0x0800000108000001ULL, 0x0800000108000001ULL, + 0x0800000108000001ULL, 0x0800000108000001ULL, +}; + +XALIGNED(16) static const word32 L_mlkem_compress_11_avx2_sllvd[] + WC_X64I_UNUSED = { + 0x0000000a, 0x00000000, 0x0000000a, 0x00000000, + 0x0000000a, 0x00000000, 0x0000000a, 0x00000000, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_11_avx2_srlvq[] + WC_X64I_UNUSED = { + 0x000000000000000aULL, 0x000000000000001eULL, + 0x000000000000000aULL, 0x000000000000001eULL, +}; + +XALIGNED(16) static const byte L_mlkem_compress_11_avx2_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_compress_11_avx2(byte* r, const sword16* p, int n) +{ + word64 rdi, rsi, rdx; + __m256i y0, y1 = _mm256_setzero_si256(), y2 = _mm256_setzero_si256(), + y3 = _mm256_setzero_si256(), y4 = _mm256_setzero_si256(), + y5 = _mm256_setzero_si256(), y6, y7, y8, y9, y10, y11, y12, y13, + y14; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + rdx = (word64)(word32)n; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_11_avx2_v, + 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_11_avx2_off, + 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx2_shift13, 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx2_mask, 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx2_shift, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx2_sllvd, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx2_srlvq, 0)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx2_shuf, 0)); + y6 = _mm256_slli_epi16(y7, 3); +L_mlkem_compress_11_avx2_start: + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y1 = _mm256_mullo_epi16(y0, y6); + y4 = _mm256_mullo_epi16(y3, y6); + y2 = _mm256_add_epi16(y0, y8); + y5 = _mm256_add_epi16(y3, y8); + y0 = _mm256_slli_epi16(y0, 3); + y3 = _mm256_slli_epi16(y3, 3); + y0 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y3, y7); + y2 = _mm256_sub_epi16(y1, y2); + y5 = _mm256_sub_epi16(y4, y5); + y1 = _mm256_andnot_si256(y1, y2); + y4 = _mm256_andnot_si256(y4, y5); + y1 = _mm256_srli_epi16(y1, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y10); + y3 = _mm256_and_si256(y3, y10); + y0 = _mm256_madd_epi16(y0, y11); + y3 = _mm256_madd_epi16(y3, y11); + y0 = _mm256_sllv_epi32(y0, y12); + y3 = _mm256_sllv_epi32(y3, y12); + y1 = _mm256_bsrli_epi128(y0, 8); + y4 = _mm256_bsrli_epi128(y3, 8); + y0 = _mm256_srlv_epi64(y0, y13); + y3 = _mm256_srlv_epi64(y3, y13); + y1 = _mm256_slli_epi64(y1, 34); + y4 = _mm256_slli_epi64(y4, 34); + y0 = _mm256_add_epi64(y0, y1); + y3 = _mm256_add_epi64(y3, y4); + y0 = _mm256_shuffle_epi8(y0, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y14))); + y3 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4), _mm256_castsi256_si128(y14))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 16), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 22), _mm256_castsi256_si128(y3)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 38), _mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y1 = _mm256_mullo_epi16(y0, y6); + y4 = _mm256_mullo_epi16(y3, y6); + y2 = _mm256_add_epi16(y0, y8); + y5 = _mm256_add_epi16(y3, y8); + y0 = _mm256_slli_epi16(y0, 3); + y3 = _mm256_slli_epi16(y3, 3); + y0 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y3, y7); + y2 = _mm256_sub_epi16(y1, y2); + y5 = _mm256_sub_epi16(y4, y5); + y1 = _mm256_andnot_si256(y1, y2); + y4 = _mm256_andnot_si256(y4, y5); + y1 = _mm256_srli_epi16(y1, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y10); + y3 = _mm256_and_si256(y3, y10); + y0 = _mm256_madd_epi16(y0, y11); + y3 = _mm256_madd_epi16(y3, y11); + y0 = _mm256_sllv_epi32(y0, y12); + y3 = _mm256_sllv_epi32(y3, y12); + y1 = _mm256_bsrli_epi128(y0, 8); + y4 = _mm256_bsrli_epi128(y3, 8); + y0 = _mm256_srlv_epi64(y0, y13); + y3 = _mm256_srlv_epi64(y3, y13); + y1 = _mm256_slli_epi64(y1, 34); + y4 = _mm256_slli_epi64(y4, 34); + y0 = _mm256_add_epi64(y0, y1); + y3 = _mm256_add_epi64(y3, y4); + y0 = _mm256_shuffle_epi8(y0, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y14))); + y3 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4), _mm256_castsi256_si128(y14))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 44), _mm256_castsi256_si128(y0)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 60), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 66), _mm256_castsi256_si128(y3)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 82), _mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y1 = _mm256_mullo_epi16(y0, y6); + y4 = _mm256_mullo_epi16(y3, y6); + y2 = _mm256_add_epi16(y0, y8); + y5 = _mm256_add_epi16(y3, y8); + y0 = _mm256_slli_epi16(y0, 3); + y3 = _mm256_slli_epi16(y3, 3); + y0 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y3, y7); + y2 = _mm256_sub_epi16(y1, y2); + y5 = _mm256_sub_epi16(y4, y5); + y1 = _mm256_andnot_si256(y1, y2); + y4 = _mm256_andnot_si256(y4, y5); + y1 = _mm256_srli_epi16(y1, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y10); + y3 = _mm256_and_si256(y3, y10); + y0 = _mm256_madd_epi16(y0, y11); + y3 = _mm256_madd_epi16(y3, y11); + y0 = _mm256_sllv_epi32(y0, y12); + y3 = _mm256_sllv_epi32(y3, y12); + y1 = _mm256_bsrli_epi128(y0, 8); + y4 = _mm256_bsrli_epi128(y3, 8); + y0 = _mm256_srlv_epi64(y0, y13); + y3 = _mm256_srlv_epi64(y3, y13); + y1 = _mm256_slli_epi64(y1, 34); + y4 = _mm256_slli_epi64(y4, 34); + y0 = _mm256_add_epi64(y0, y1); + y3 = _mm256_add_epi64(y3, y4); + y0 = _mm256_shuffle_epi8(y0, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y14))); + y3 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4), _mm256_castsi256_si128(y14))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 88), _mm256_castsi256_si128(y0)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 104), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 110), _mm256_castsi256_si128(y3)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 126), _mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y1 = _mm256_mullo_epi16(y0, y6); + y4 = _mm256_mullo_epi16(y3, y6); + y2 = _mm256_add_epi16(y0, y8); + y5 = _mm256_add_epi16(y3, y8); + y0 = _mm256_slli_epi16(y0, 3); + y3 = _mm256_slli_epi16(y3, 3); + y0 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y3, y7); + y2 = _mm256_sub_epi16(y1, y2); + y5 = _mm256_sub_epi16(y4, y5); + y1 = _mm256_andnot_si256(y1, y2); + y4 = _mm256_andnot_si256(y4, y5); + y1 = _mm256_srli_epi16(y1, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y10); + y3 = _mm256_and_si256(y3, y10); + y0 = _mm256_madd_epi16(y0, y11); + y3 = _mm256_madd_epi16(y3, y11); + y0 = _mm256_sllv_epi32(y0, y12); + y3 = _mm256_sllv_epi32(y3, y12); + y1 = _mm256_bsrli_epi128(y0, 8); + y4 = _mm256_bsrli_epi128(y3, 8); + y0 = _mm256_srlv_epi64(y0, y13); + y3 = _mm256_srlv_epi64(y3, y13); + y1 = _mm256_slli_epi64(y1, 34); + y4 = _mm256_slli_epi64(y4, 34); + y0 = _mm256_add_epi64(y0, y1); + y3 = _mm256_add_epi64(y3, y4); + y0 = _mm256_shuffle_epi8(y0, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y14))); + y3 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4), _mm256_castsi256_si128(y14))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 132), _mm256_castsi256_si128(y0)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 148), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 154), _mm256_castsi256_si128(y3)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 170), _mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y1 = _mm256_mullo_epi16(y0, y6); + y4 = _mm256_mullo_epi16(y3, y6); + y2 = _mm256_add_epi16(y0, y8); + y5 = _mm256_add_epi16(y3, y8); + y0 = _mm256_slli_epi16(y0, 3); + y3 = _mm256_slli_epi16(y3, 3); + y0 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y3, y7); + y2 = _mm256_sub_epi16(y1, y2); + y5 = _mm256_sub_epi16(y4, y5); + y1 = _mm256_andnot_si256(y1, y2); + y4 = _mm256_andnot_si256(y4, y5); + y1 = _mm256_srli_epi16(y1, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y10); + y3 = _mm256_and_si256(y3, y10); + y0 = _mm256_madd_epi16(y0, y11); + y3 = _mm256_madd_epi16(y3, y11); + y0 = _mm256_sllv_epi32(y0, y12); + y3 = _mm256_sllv_epi32(y3, y12); + y1 = _mm256_bsrli_epi128(y0, 8); + y4 = _mm256_bsrli_epi128(y3, 8); + y0 = _mm256_srlv_epi64(y0, y13); + y3 = _mm256_srlv_epi64(y3, y13); + y1 = _mm256_slli_epi64(y1, 34); + y4 = _mm256_slli_epi64(y4, 34); + y0 = _mm256_add_epi64(y0, y1); + y3 = _mm256_add_epi64(y3, y4); + y0 = _mm256_shuffle_epi8(y0, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y14))); + y3 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4), _mm256_castsi256_si128(y14))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 176), _mm256_castsi256_si128(y0)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 192), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 198), _mm256_castsi256_si128(y3)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 214), _mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y1 = _mm256_mullo_epi16(y0, y6); + y4 = _mm256_mullo_epi16(y3, y6); + y2 = _mm256_add_epi16(y0, y8); + y5 = _mm256_add_epi16(y3, y8); + y0 = _mm256_slli_epi16(y0, 3); + y3 = _mm256_slli_epi16(y3, 3); + y0 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y3, y7); + y2 = _mm256_sub_epi16(y1, y2); + y5 = _mm256_sub_epi16(y4, y5); + y1 = _mm256_andnot_si256(y1, y2); + y4 = _mm256_andnot_si256(y4, y5); + y1 = _mm256_srli_epi16(y1, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y10); + y3 = _mm256_and_si256(y3, y10); + y0 = _mm256_madd_epi16(y0, y11); + y3 = _mm256_madd_epi16(y3, y11); + y0 = _mm256_sllv_epi32(y0, y12); + y3 = _mm256_sllv_epi32(y3, y12); + y1 = _mm256_bsrli_epi128(y0, 8); + y4 = _mm256_bsrli_epi128(y3, 8); + y0 = _mm256_srlv_epi64(y0, y13); + y3 = _mm256_srlv_epi64(y3, y13); + y1 = _mm256_slli_epi64(y1, 34); + y4 = _mm256_slli_epi64(y4, 34); + y0 = _mm256_add_epi64(y0, y1); + y3 = _mm256_add_epi64(y3, y4); + y0 = _mm256_shuffle_epi8(y0, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y14))); + y3 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4), _mm256_castsi256_si128(y14))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 220), _mm256_castsi256_si128(y0)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 236), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 242), _mm256_castsi256_si128(y3)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 258), _mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y1 = _mm256_mullo_epi16(y0, y6); + y4 = _mm256_mullo_epi16(y3, y6); + y2 = _mm256_add_epi16(y0, y8); + y5 = _mm256_add_epi16(y3, y8); + y0 = _mm256_slli_epi16(y0, 3); + y3 = _mm256_slli_epi16(y3, 3); + y0 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y3, y7); + y2 = _mm256_sub_epi16(y1, y2); + y5 = _mm256_sub_epi16(y4, y5); + y1 = _mm256_andnot_si256(y1, y2); + y4 = _mm256_andnot_si256(y4, y5); + y1 = _mm256_srli_epi16(y1, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y10); + y3 = _mm256_and_si256(y3, y10); + y0 = _mm256_madd_epi16(y0, y11); + y3 = _mm256_madd_epi16(y3, y11); + y0 = _mm256_sllv_epi32(y0, y12); + y3 = _mm256_sllv_epi32(y3, y12); + y1 = _mm256_bsrli_epi128(y0, 8); + y4 = _mm256_bsrli_epi128(y3, 8); + y0 = _mm256_srlv_epi64(y0, y13); + y3 = _mm256_srlv_epi64(y3, y13); + y1 = _mm256_slli_epi64(y1, 34); + y4 = _mm256_slli_epi64(y4, 34); + y0 = _mm256_add_epi64(y0, y1); + y3 = _mm256_add_epi64(y3, y4); + y0 = _mm256_shuffle_epi8(y0, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y14))); + y3 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4), _mm256_castsi256_si128(y14))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 264), _mm256_castsi256_si128(y0)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 280), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 286), _mm256_castsi256_si128(y3)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 302), _mm256_castsi256_si128(y4)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y1 = _mm256_mullo_epi16(y0, y6); + y4 = _mm256_mullo_epi16(y3, y6); + y2 = _mm256_add_epi16(y0, y8); + y5 = _mm256_add_epi16(y3, y8); + y0 = _mm256_slli_epi16(y0, 3); + y3 = _mm256_slli_epi16(y3, 3); + y0 = _mm256_mulhi_epi16(y0, y7); + y3 = _mm256_mulhi_epi16(y3, y7); + y2 = _mm256_sub_epi16(y1, y2); + y5 = _mm256_sub_epi16(y4, y5); + y1 = _mm256_andnot_si256(y1, y2); + y4 = _mm256_andnot_si256(y4, y5); + y1 = _mm256_srli_epi16(y1, 15); + y4 = _mm256_srli_epi16(y4, 15); + y0 = _mm256_sub_epi16(y0, y1); + y3 = _mm256_sub_epi16(y3, y4); + y0 = _mm256_mulhrs_epi16(y0, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y10); + y3 = _mm256_and_si256(y3, y10); + y0 = _mm256_madd_epi16(y0, y11); + y3 = _mm256_madd_epi16(y3, y11); + y0 = _mm256_sllv_epi32(y0, y12); + y3 = _mm256_sllv_epi32(y3, y12); + y1 = _mm256_bsrli_epi128(y0, 8); + y4 = _mm256_bsrli_epi128(y3, 8); + y0 = _mm256_srlv_epi64(y0, y13); + y3 = _mm256_srlv_epi64(y3, y13); + y1 = _mm256_slli_epi64(y1, 34); + y4 = _mm256_slli_epi64(y4, 34); + y0 = _mm256_add_epi64(y0, y1); + y3 = _mm256_add_epi64(y3, y4); + y0 = _mm256_shuffle_epi8(y0, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y4 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y3, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y14))); + y3 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y3), + _mm256_castsi256_si128(y4), _mm256_castsi256_si128(y14))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 308), _mm256_castsi256_si128(y0)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 324), _mm256_castsi256_si128(y1)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 330), _mm256_castsi256_si128(y3)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 346), _mm256_castsi256_si128(y4)); + rdi = (word64)(rdi + 0x160); + rsi = (word64)(rsi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_compress_11_avx2_start; + } +} + +XALIGNED(16) static const word16 L_mlkem_decompress_11_avx2_q[] + WC_X64I_UNUSED = { + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_11_avx2_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x04, 0x05, + 0x05, 0x06, 0x06, 0x07, 0x08, 0x09, 0x09, 0x0a, + 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x07, 0x08, + 0x08, 0x09, 0x09, 0x0a, 0x0b, 0x0c, 0x0c, 0x0d, +}; + +XALIGNED(16) static const word32 L_mlkem_decompress_11_avx2_sllv[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000000, 0x00000000, + 0x00000000, 0x00000001, 0x00000000, 0x00000000, +}; + +XALIGNED(32) static const word64 L_mlkem_decompress_11_avx2_srlv[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_11_avx2_shift[] + WC_X64I_UNUSED = { + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_11_avx2_mask[] + WC_X64I_UNUSED = { + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_decompress_11_avx2(sword16* p, const byte* r, int n) +{ + word64 rdi, rsi, rdx; + __m256i y0 = _mm256_setzero_si256(), y1 = _mm256_setzero_si256(), + y2 = _mm256_setzero_si256(), y3 = _mm256_setzero_si256(), y4, y5, + y6, y7, y8, y9; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + rdx = (word64)(word32)n; + + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_decompress_11_avx2_q, + 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_11_avx2_shuf, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_11_avx2_sllv, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_11_avx2_srlv, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_11_avx2_shift, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_11_avx2_mask, 0)); +L_mlkem_decompress_11_avx2_start: + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 22)), 0x94); + y2 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 44)), 0x94); + y3 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 66)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y5); + y1 = _mm256_shuffle_epi8(y1, y5); + y2 = _mm256_shuffle_epi8(y2, y5); + y3 = _mm256_shuffle_epi8(y3, y5); + y0 = _mm256_srlv_epi32(y0, y6); + y1 = _mm256_srlv_epi32(y1, y6); + y2 = _mm256_srlv_epi32(y2, y6); + y3 = _mm256_srlv_epi32(y3, y6); + y0 = _mm256_srlv_epi64(y0, y7); + y1 = _mm256_srlv_epi64(y1, y7); + y2 = _mm256_srlv_epi64(y2, y7); + y3 = _mm256_srlv_epi64(y3, y7); + y0 = _mm256_mullo_epi16(y0, y8); + y1 = _mm256_mullo_epi16(y1, y8); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_srli_epi16(y0, 1); + y1 = _mm256_srli_epi16(y1, 1); + y2 = _mm256_srli_epi16(y2, 1); + y3 = _mm256_srli_epi16(y3, 1); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y2 = _mm256_and_si256(y2, y9); + y3 = _mm256_and_si256(y3, y9); + y0 = _mm256_mulhrs_epi16(y0, y4); + y1 = _mm256_mulhrs_epi16(y1, y4); + y2 = _mm256_mulhrs_epi16(y2, y4); + y3 = _mm256_mulhrs_epi16(y3, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 88)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 110)), 0x94); + y2 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 132)), 0x94); + y3 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 154)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y5); + y1 = _mm256_shuffle_epi8(y1, y5); + y2 = _mm256_shuffle_epi8(y2, y5); + y3 = _mm256_shuffle_epi8(y3, y5); + y0 = _mm256_srlv_epi32(y0, y6); + y1 = _mm256_srlv_epi32(y1, y6); + y2 = _mm256_srlv_epi32(y2, y6); + y3 = _mm256_srlv_epi32(y3, y6); + y0 = _mm256_srlv_epi64(y0, y7); + y1 = _mm256_srlv_epi64(y1, y7); + y2 = _mm256_srlv_epi64(y2, y7); + y3 = _mm256_srlv_epi64(y3, y7); + y0 = _mm256_mullo_epi16(y0, y8); + y1 = _mm256_mullo_epi16(y1, y8); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_srli_epi16(y0, 1); + y1 = _mm256_srli_epi16(y1, 1); + y2 = _mm256_srli_epi16(y2, 1); + y3 = _mm256_srli_epi16(y3, 1); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y2 = _mm256_and_si256(y2, y9); + y3 = _mm256_and_si256(y3, y9); + y0 = _mm256_mulhrs_epi16(y0, y4); + y1 = _mm256_mulhrs_epi16(y1, y4); + y2 = _mm256_mulhrs_epi16(y2, y4); + y3 = _mm256_mulhrs_epi16(y3, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y3); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 176)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 198)), 0x94); + y2 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 220)), 0x94); + y3 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 242)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y5); + y1 = _mm256_shuffle_epi8(y1, y5); + y2 = _mm256_shuffle_epi8(y2, y5); + y3 = _mm256_shuffle_epi8(y3, y5); + y0 = _mm256_srlv_epi32(y0, y6); + y1 = _mm256_srlv_epi32(y1, y6); + y2 = _mm256_srlv_epi32(y2, y6); + y3 = _mm256_srlv_epi32(y3, y6); + y0 = _mm256_srlv_epi64(y0, y7); + y1 = _mm256_srlv_epi64(y1, y7); + y2 = _mm256_srlv_epi64(y2, y7); + y3 = _mm256_srlv_epi64(y3, y7); + y0 = _mm256_mullo_epi16(y0, y8); + y1 = _mm256_mullo_epi16(y1, y8); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_srli_epi16(y0, 1); + y1 = _mm256_srli_epi16(y1, 1); + y2 = _mm256_srli_epi16(y2, 1); + y3 = _mm256_srli_epi16(y3, 1); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y2 = _mm256_and_si256(y2, y9); + y3 = _mm256_and_si256(y3, y9); + y0 = _mm256_mulhrs_epi16(y0, y4); + y1 = _mm256_mulhrs_epi16(y1, y4); + y2 = _mm256_mulhrs_epi16(y2, y4); + y3 = _mm256_mulhrs_epi16(y3, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + y0 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 264)), 0x94); + y1 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 286)), 0x94); + y2 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 308)), 0x94); + y3 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 330)), 0x94); + y0 = _mm256_shuffle_epi8(y0, y5); + y1 = _mm256_shuffle_epi8(y1, y5); + y2 = _mm256_shuffle_epi8(y2, y5); + y3 = _mm256_shuffle_epi8(y3, y5); + y0 = _mm256_srlv_epi32(y0, y6); + y1 = _mm256_srlv_epi32(y1, y6); + y2 = _mm256_srlv_epi32(y2, y6); + y3 = _mm256_srlv_epi32(y3, y6); + y0 = _mm256_srlv_epi64(y0, y7); + y1 = _mm256_srlv_epi64(y1, y7); + y2 = _mm256_srlv_epi64(y2, y7); + y3 = _mm256_srlv_epi64(y3, y7); + y0 = _mm256_mullo_epi16(y0, y8); + y1 = _mm256_mullo_epi16(y1, y8); + y2 = _mm256_mullo_epi16(y2, y8); + y3 = _mm256_mullo_epi16(y3, y8); + y0 = _mm256_srli_epi16(y0, 1); + y1 = _mm256_srli_epi16(y1, 1); + y2 = _mm256_srli_epi16(y2, 1); + y3 = _mm256_srli_epi16(y3, 1); + y0 = _mm256_and_si256(y0, y9); + y1 = _mm256_and_si256(y1, y9); + y2 = _mm256_and_si256(y2, y9); + y3 = _mm256_and_si256(y3, y9); + y0 = _mm256_mulhrs_epi16(y0, y4); + y1 = _mm256_mulhrs_epi16(y1, y4); + y2 = _mm256_mulhrs_epi16(y2, y4); + y3 = _mm256_mulhrs_epi16(y3, y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); + rsi = (word64)(rsi + 0x160); + rdi = (word64)(rdi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_decompress_11_avx2_start; + } +} + +XALIGNED(16) static const word16 L_mlkem_compress_4_avx2_mask[] + WC_X64I_UNUSED = { + 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, + 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, 0x000f, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_4_avx2_shift[] + WC_X64I_UNUSED = { + 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, + 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, +}; + +XALIGNED(16) static const word32 L_mlkem_compress_4_avx2_perm[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000001, 0x00000005, + 0x00000002, 0x00000006, 0x00000003, 0x00000007, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_4_avx2_v[] WC_X64I_UNUSED = { + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_4_avx2_shift12[] + WC_X64I_UNUSED = { + 0x1001, 0x1001, 0x1001, 0x1001, 0x1001, 0x1001, 0x1001, 0x1001, + 0x1001, 0x1001, 0x1001, 0x1001, 0x1001, 0x1001, 0x1001, 0x1001, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_compress_4_avx2(byte* b, const sword16* p) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12; + + rdi = (word64)(size_t)b; + rsi = (word64)(size_t)p; + + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_4_avx2_mask, + 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_4_avx2_shift, + 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_4_avx2_perm, + 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_4_avx2_v, + 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_4_avx2_shift12, 0)); + y0 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y1 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + y2 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y3 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + y0 = _mm256_mulhrs_epi16(y0, y9); + y1 = _mm256_mulhrs_epi16(y1, y9); + y2 = _mm256_mulhrs_epi16(y2, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_packus_epi16(y0, y1); + y2 = _mm256_packus_epi16(y2, y3); + y0 = _mm256_maddubs_epi16(y0, y12); + y2 = _mm256_maddubs_epi16(y2, y12); + y0 = _mm256_packus_epi16(y0, y2); + y4 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + y5 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + y6 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + y7 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + y4 = _mm256_mulhrs_epi16(y4, y9); + y5 = _mm256_mulhrs_epi16(y5, y9); + y6 = _mm256_mulhrs_epi16(y6, y9); + y7 = _mm256_mulhrs_epi16(y7, y9); + y4 = _mm256_and_si256(y4, y8); + y5 = _mm256_and_si256(y5, y8); + y6 = _mm256_and_si256(y6, y8); + y7 = _mm256_and_si256(y7, y8); + y4 = _mm256_packus_epi16(y4, y5); + y6 = _mm256_packus_epi16(y6, y7); + y4 = _mm256_maddubs_epi16(y4, y12); + y6 = _mm256_maddubs_epi16(y6, y12); + y4 = _mm256_packus_epi16(y4, y6); + y0 = _mm256_permutevar8x32_epi32(y0, y10); + y4 = _mm256_permutevar8x32_epi32(y4, y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y4); + y0 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + y1 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + y2 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320))); + y3 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352))); + y0 = _mm256_mulhrs_epi16(y0, y9); + y1 = _mm256_mulhrs_epi16(y1, y9); + y2 = _mm256_mulhrs_epi16(y2, y9); + y3 = _mm256_mulhrs_epi16(y3, y9); + y0 = _mm256_and_si256(y0, y8); + y1 = _mm256_and_si256(y1, y8); + y2 = _mm256_and_si256(y2, y8); + y3 = _mm256_and_si256(y3, y8); + y0 = _mm256_packus_epi16(y0, y1); + y2 = _mm256_packus_epi16(y2, y3); + y0 = _mm256_maddubs_epi16(y0, y12); + y2 = _mm256_maddubs_epi16(y2, y12); + y0 = _mm256_packus_epi16(y0, y2); + y4 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + y5 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + y6 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448))); + y7 = _mm256_mulhi_epi16(y11, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480))); + y4 = _mm256_mulhrs_epi16(y4, y9); + y5 = _mm256_mulhrs_epi16(y5, y9); + y6 = _mm256_mulhrs_epi16(y6, y9); + y7 = _mm256_mulhrs_epi16(y7, y9); + y4 = _mm256_and_si256(y4, y8); + y5 = _mm256_and_si256(y5, y8); + y6 = _mm256_and_si256(y6, y8); + y7 = _mm256_and_si256(y7, y8); + y4 = _mm256_packus_epi16(y4, y5); + y6 = _mm256_packus_epi16(y6, y7); + y4 = _mm256_maddubs_epi16(y4, y12); + y6 = _mm256_maddubs_epi16(y6, y12); + y4 = _mm256_packus_epi16(y4, y6); + y0 = _mm256_permutevar8x32_epi32(y0, y10); + y4 = _mm256_permutevar8x32_epi32(y4, y10); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y4); +} + +XALIGNED(16) static const word32 L_mlkem_decompress_4_avx2_mask[] + WC_X64I_UNUSED = { + 0x00f0000f, 0x00f0000f, 0x00f0000f, 0x00f0000f, + 0x00f0000f, 0x00f0000f, 0x00f0000f, 0x00f0000f, +}; + +XALIGNED(16) static const word32 L_mlkem_decompress_4_avx2_shift[] + WC_X64I_UNUSED = { + 0x00800800, 0x00800800, 0x00800800, 0x00800800, + 0x00800800, 0x00800800, 0x00800800, 0x00800800, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_4_avx2_q[] + WC_X64I_UNUSED = { + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_4_avx2_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, + 0x04, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x05, + 0x06, 0x06, 0x06, 0x06, 0x07, 0x07, 0x07, 0x07, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_decompress_4_avx2(sword16* p, const byte* r) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + + y4 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_4_avx2_mask, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_4_avx2_shift, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_4_avx2_shuf, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_decompress_4_avx2_q, + 0)); + y0 = _mm256_set1_epi64x((long long)WC_L64(rsi, 0)); + y1 = _mm256_set1_epi64x((long long)WC_L64(rsi, 8)); + y2 = _mm256_set1_epi64x((long long)WC_L64(rsi, 16)); + y3 = _mm256_set1_epi64x((long long)WC_L64(rsi, 24)); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_shuffle_epi8(y3, y6); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_mullo_epi16(y0, y5); + y1 = _mm256_mullo_epi16(y1, y5); + y2 = _mm256_mullo_epi16(y2, y5); + y3 = _mm256_mullo_epi16(y3, y5); + y0 = _mm256_mulhrs_epi16(y0, y7); + y1 = _mm256_mulhrs_epi16(y1, y7); + y2 = _mm256_mulhrs_epi16(y2, y7); + y3 = _mm256_mulhrs_epi16(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + y0 = _mm256_set1_epi64x((long long)WC_L64(rsi, 32)); + y1 = _mm256_set1_epi64x((long long)WC_L64(rsi, 40)); + y2 = _mm256_set1_epi64x((long long)WC_L64(rsi, 48)); + y3 = _mm256_set1_epi64x((long long)WC_L64(rsi, 56)); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_shuffle_epi8(y3, y6); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_mullo_epi16(y0, y5); + y1 = _mm256_mullo_epi16(y1, y5); + y2 = _mm256_mullo_epi16(y2, y5); + y3 = _mm256_mullo_epi16(y3, y5); + y0 = _mm256_mulhrs_epi16(y0, y7); + y1 = _mm256_mulhrs_epi16(y1, y7); + y2 = _mm256_mulhrs_epi16(y2, y7); + y3 = _mm256_mulhrs_epi16(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y3); + y0 = _mm256_set1_epi64x((long long)WC_L64(rsi, 64)); + y1 = _mm256_set1_epi64x((long long)WC_L64(rsi, 72)); + y2 = _mm256_set1_epi64x((long long)WC_L64(rsi, 80)); + y3 = _mm256_set1_epi64x((long long)WC_L64(rsi, 88)); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_shuffle_epi8(y3, y6); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_mullo_epi16(y0, y5); + y1 = _mm256_mullo_epi16(y1, y5); + y2 = _mm256_mullo_epi16(y2, y5); + y3 = _mm256_mullo_epi16(y3, y5); + y0 = _mm256_mulhrs_epi16(y0, y7); + y1 = _mm256_mulhrs_epi16(y1, y7); + y2 = _mm256_mulhrs_epi16(y2, y7); + y3 = _mm256_mulhrs_epi16(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + y0 = _mm256_set1_epi64x((long long)WC_L64(rsi, 96)); + y1 = _mm256_set1_epi64x((long long)WC_L64(rsi, 104)); + y2 = _mm256_set1_epi64x((long long)WC_L64(rsi, 112)); + y3 = _mm256_set1_epi64x((long long)WC_L64(rsi, 120)); + y0 = _mm256_shuffle_epi8(y0, y6); + y1 = _mm256_shuffle_epi8(y1, y6); + y2 = _mm256_shuffle_epi8(y2, y6); + y3 = _mm256_shuffle_epi8(y3, y6); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y2 = _mm256_and_si256(y2, y4); + y3 = _mm256_and_si256(y3, y4); + y0 = _mm256_mullo_epi16(y0, y5); + y1 = _mm256_mullo_epi16(y1, y5); + y2 = _mm256_mullo_epi16(y2, y5); + y3 = _mm256_mullo_epi16(y3, y5); + y0 = _mm256_mulhrs_epi16(y0, y7); + y1 = _mm256_mulhrs_epi16(y1, y7); + y2 = _mm256_mulhrs_epi16(y2, y7); + y3 = _mm256_mulhrs_epi16(y3, y7); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y3); +} + +XALIGNED(16) static const word16 L_mlkem_compress_5_avx2_v[] WC_X64I_UNUSED = { + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, + 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, 0x4ebf, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_5_avx2_shift[] + WC_X64I_UNUSED = { + 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, + 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, 0x0400, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_5_avx2_mask[] + WC_X64I_UNUSED = { + 0x001f, 0x001f, 0x001f, 0x001f, 0x001f, 0x001f, 0x001f, 0x001f, + 0x001f, 0x001f, 0x001f, 0x001f, 0x001f, 0x001f, 0x001f, 0x001f, +}; + +XALIGNED(16) static const word16 L_mlkem_compress_5_avx2_shift1[] + WC_X64I_UNUSED = { + 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, + 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, +}; + +XALIGNED(16) static const word32 L_mlkem_compress_5_avx2_shift2[] + WC_X64I_UNUSED = { + 0x04000001, 0x04000001, 0x04000001, 0x04000001, + 0x04000001, 0x04000001, 0x04000001, 0x04000001, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_5_avx2_shlv[] + WC_X64I_UNUSED = { + 0x000000000000000cULL, 0x000000000000000cULL, + 0x000000000000000cULL, 0x000000000000000cULL, +}; + +XALIGNED(16) static const byte L_mlkem_compress_5_avx2_shuffle[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0xff, + 0x09, 0x0a, 0x0b, 0x0c, 0xff, 0x00, 0x01, 0x02, + 0x03, 0x04, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_compress_5_avx2(byte* b, const sword16* p) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8; + + rdi = (word64)(size_t)b; + rsi = (word64)(size_t)p; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_5_avx2_v, + 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_5_avx2_shift, + 0)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_5_avx2_mask, + 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx2_shift1, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx2_shift2, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_compress_5_avx2_shlv, + 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx2_shuffle, 0)); + y0 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y1 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + y0 = _mm256_mulhrs_epi16(y0, y3); + y1 = _mm256_mulhrs_epi16(y1, y3); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_maddubs_epi16(y0, y5); + y0 = _mm256_madd_epi16(y0, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y0 = _mm256_srlv_epi64(y0, y7); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y0)); + WC_S32(rdi, 16) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y1)); + y0 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y1 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + y0 = _mm256_mulhrs_epi16(y0, y3); + y1 = _mm256_mulhrs_epi16(y1, y3); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_maddubs_epi16(y0, y5); + y0 = _mm256_madd_epi16(y0, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y0 = _mm256_srlv_epi64(y0, y7); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 20), _mm256_castsi256_si128(y0)); + WC_S32(rdi, 36) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y1)); + y0 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + y1 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + y0 = _mm256_mulhrs_epi16(y0, y3); + y1 = _mm256_mulhrs_epi16(y1, y3); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_maddubs_epi16(y0, y5); + y0 = _mm256_madd_epi16(y0, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y0 = _mm256_srlv_epi64(y0, y7); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 40), _mm256_castsi256_si128(y0)); + WC_S32(rdi, 56) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y1)); + y0 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + y1 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + y0 = _mm256_mulhrs_epi16(y0, y3); + y1 = _mm256_mulhrs_epi16(y1, y3); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_maddubs_epi16(y0, y5); + y0 = _mm256_madd_epi16(y0, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y0 = _mm256_srlv_epi64(y0, y7); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 60), _mm256_castsi256_si128(y0)); + WC_S32(rdi, 76) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y1)); + y0 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + y1 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + y0 = _mm256_mulhrs_epi16(y0, y3); + y1 = _mm256_mulhrs_epi16(y1, y3); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_maddubs_epi16(y0, y5); + y0 = _mm256_madd_epi16(y0, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y0 = _mm256_srlv_epi64(y0, y7); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), _mm256_castsi256_si128(y0)); + WC_S32(rdi, 96) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y1)); + y0 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320))); + y1 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352))); + y0 = _mm256_mulhrs_epi16(y0, y3); + y1 = _mm256_mulhrs_epi16(y1, y3); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_maddubs_epi16(y0, y5); + y0 = _mm256_madd_epi16(y0, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y0 = _mm256_srlv_epi64(y0, y7); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 100), _mm256_castsi256_si128(y0)); + WC_S32(rdi, 116) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y1)); + y0 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + y1 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + y0 = _mm256_mulhrs_epi16(y0, y3); + y1 = _mm256_mulhrs_epi16(y1, y3); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_maddubs_epi16(y0, y5); + y0 = _mm256_madd_epi16(y0, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y0 = _mm256_srlv_epi64(y0, y7); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 120), _mm256_castsi256_si128(y0)); + WC_S32(rdi, 136) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y1)); + y0 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448))); + y1 = _mm256_mulhi_epi16(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480))); + y0 = _mm256_mulhrs_epi16(y0, y3); + y1 = _mm256_mulhrs_epi16(y1, y3); + y0 = _mm256_and_si256(y0, y4); + y1 = _mm256_and_si256(y1, y4); + y0 = _mm256_packus_epi16(y0, y1); + y0 = _mm256_maddubs_epi16(y0, y5); + y0 = _mm256_madd_epi16(y0, y6); + y0 = _mm256_sllv_epi32(y0, y7); + y0 = _mm256_srlv_epi64(y0, y7); + y0 = _mm256_shuffle_epi8(y0, y8); + y1 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y0, 1)); + y0 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y0), + _mm256_castsi256_si128(y1), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 140), _mm256_castsi256_si128(y0)); + WC_S32(rdi, 156) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y1)); +} + +XALIGNED(16) static const word16 L_mlkem_decompress_5_avx2_q[] + WC_X64I_UNUSED = { + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_5_avx2_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, + 0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, + 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x06, 0x07, + 0x07, 0x08, 0x08, 0x08, 0x08, 0x09, 0x09, 0x09, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_5_avx2_mask[] + WC_X64I_UNUSED = { + 0x001f, 0x03e0, 0x007c, 0x0f80, 0x01f0, 0x003e, 0x07c0, 0x00f8, + 0x001f, 0x03e0, 0x007c, 0x0f80, 0x01f0, 0x003e, 0x07c0, 0x00f8, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_5_avx2_shift[] + WC_X64I_UNUSED = { + 0x0400, 0x0020, 0x0100, 0x0008, 0x0040, 0x0200, 0x0010, 0x0080, + 0x0400, 0x0020, 0x0100, 0x0008, 0x0040, 0x0200, 0x0010, 0x0080, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_decompress_5_avx2(sword16* p, const byte* r) +{ + word64 rdi, rsi, rdx; + __m256i y0, y1, y2, y3, y4; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_decompress_5_avx2_q, + 0)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_5_avx2_shuf, 0)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_5_avx2_mask, 0)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_decompress_5_avx2_shift, 0)); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 0))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 10))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 20))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 30))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 40))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 50))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 60))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 70))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 80))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 90))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 100))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 110))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 120))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 130))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y0); + y0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 140))); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y0); + y0 = _mm256_zextsi128_si256(_mm_loadl_epi64((const __m128i*)WC_PR(rsi, + 150))); + rdx = (word64)((word64)WC_L16(rsi, 158)); + y0 = _mm256_zextsi128_si256(_mm_insert_epi64(_mm256_castsi256_si128(y0), ( + long long)rdx, 1)); + y0 = _mm256_inserti128_si256(y0, _mm256_castsi256_si128(y0), 1); + y0 = _mm256_shuffle_epi8(y0, y2); + y0 = _mm256_and_si256(y0, y3); + y0 = _mm256_mullo_epi16(y0, y4); + y0 = _mm256_mulhrs_epi16(y0, y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y0); +} + +XALIGNED(16) static const word32 L_mlkem_from_msg_avx2_shift[] + WC_X64I_UNUSED = { + 0x00000003, 0x00000002, 0x00000001, 0x00000000, + 0x00000003, 0x00000002, 0x00000001, 0x00000000, +}; + +XALIGNED(16) static const byte L_mlkem_from_msg_avx2_shuf[] WC_X64I_UNUSED = { + 0x00, 0x01, 0x04, 0x05, 0x08, 0x09, 0x0c, 0x0d, + 0x02, 0x03, 0x06, 0x07, 0x0a, 0x0b, 0x0e, 0x0f, + 0x00, 0x01, 0x04, 0x05, 0x08, 0x09, 0x0c, 0x0d, + 0x02, 0x03, 0x06, 0x07, 0x0a, 0x0b, 0x0e, 0x0f, +}; + +XALIGNED(16) static const word16 L_mlkem_from_msg_avx2_hqs[] WC_X64I_UNUSED = { + 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, + 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_from_msg_avx2(sword16* p, const byte* m) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)m; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_from_msg_avx2_shift, + 0)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_from_msg_avx2_shuf, + 0)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_from_msg_avx2_hqs, + 0)); + y4 = _mm256_shuffle_epi32(y0, 0); + y4 = _mm256_sllv_epi32(y4, y9); + y4 = _mm256_shuffle_epi8(y4, y10); + y1 = _mm256_slli_epi16(y4, 12); + y2 = _mm256_slli_epi16(y4, 8); + y3 = _mm256_slli_epi16(y4, 4); + y1 = _mm256_srai_epi16(y1, 15); + y2 = _mm256_srai_epi16(y2, 15); + y3 = _mm256_srai_epi16(y3, 15); + y4 = _mm256_srai_epi16(y4, 15); + y1 = _mm256_and_si256(y1, y11); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y4 = _mm256_and_si256(y4, y11); + y5 = _mm256_unpacklo_epi64(y1, y2); + y7 = _mm256_unpackhi_epi64(y1, y2); + y6 = _mm256_unpacklo_epi64(y3, y4); + y8 = _mm256_unpackhi_epi64(y3, y4); + y1 = _mm256_permute2x128_si256(y5, y6, 0x20); + y3 = _mm256_permute2x128_si256(y5, y6, 0x31); + y2 = _mm256_permute2x128_si256(y7, y8, 0x20); + y4 = _mm256_permute2x128_si256(y7, y8, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y4); + y4 = _mm256_shuffle_epi32(y0, 0x55); + y4 = _mm256_sllv_epi32(y4, y9); + y4 = _mm256_shuffle_epi8(y4, y10); + y1 = _mm256_slli_epi16(y4, 12); + y2 = _mm256_slli_epi16(y4, 8); + y3 = _mm256_slli_epi16(y4, 4); + y1 = _mm256_srai_epi16(y1, 15); + y2 = _mm256_srai_epi16(y2, 15); + y3 = _mm256_srai_epi16(y3, 15); + y4 = _mm256_srai_epi16(y4, 15); + y1 = _mm256_and_si256(y1, y11); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y4 = _mm256_and_si256(y4, y11); + y5 = _mm256_unpacklo_epi64(y1, y2); + y7 = _mm256_unpackhi_epi64(y1, y2); + y6 = _mm256_unpacklo_epi64(y3, y4); + y8 = _mm256_unpackhi_epi64(y3, y4); + y1 = _mm256_permute2x128_si256(y5, y6, 0x20); + y3 = _mm256_permute2x128_si256(y5, y6, 0x31); + y2 = _mm256_permute2x128_si256(y7, y8, 0x20); + y4 = _mm256_permute2x128_si256(y7, y8, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y4); + y4 = _mm256_shuffle_epi32(y0, 0xaa); + y4 = _mm256_sllv_epi32(y4, y9); + y4 = _mm256_shuffle_epi8(y4, y10); + y1 = _mm256_slli_epi16(y4, 12); + y2 = _mm256_slli_epi16(y4, 8); + y3 = _mm256_slli_epi16(y4, 4); + y1 = _mm256_srai_epi16(y1, 15); + y2 = _mm256_srai_epi16(y2, 15); + y3 = _mm256_srai_epi16(y3, 15); + y4 = _mm256_srai_epi16(y4, 15); + y1 = _mm256_and_si256(y1, y11); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y4 = _mm256_and_si256(y4, y11); + y5 = _mm256_unpacklo_epi64(y1, y2); + y7 = _mm256_unpackhi_epi64(y1, y2); + y6 = _mm256_unpacklo_epi64(y3, y4); + y8 = _mm256_unpackhi_epi64(y3, y4); + y1 = _mm256_permute2x128_si256(y5, y6, 0x20); + y3 = _mm256_permute2x128_si256(y5, y6, 0x31); + y2 = _mm256_permute2x128_si256(y7, y8, 0x20); + y4 = _mm256_permute2x128_si256(y7, y8, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y4); + y4 = _mm256_shuffle_epi32(y0, 0xff); + y4 = _mm256_sllv_epi32(y4, y9); + y4 = _mm256_shuffle_epi8(y4, y10); + y1 = _mm256_slli_epi16(y4, 12); + y2 = _mm256_slli_epi16(y4, 8); + y3 = _mm256_slli_epi16(y4, 4); + y1 = _mm256_srai_epi16(y1, 15); + y2 = _mm256_srai_epi16(y2, 15); + y3 = _mm256_srai_epi16(y3, 15); + y4 = _mm256_srai_epi16(y4, 15); + y1 = _mm256_and_si256(y1, y11); + y2 = _mm256_and_si256(y2, y11); + y3 = _mm256_and_si256(y3, y11); + y4 = _mm256_and_si256(y4, y11); + y5 = _mm256_unpacklo_epi64(y1, y2); + y7 = _mm256_unpackhi_epi64(y1, y2); + y6 = _mm256_unpacklo_epi64(y3, y4); + y8 = _mm256_unpackhi_epi64(y3, y4); + y1 = _mm256_permute2x128_si256(y5, y6, 0x20); + y3 = _mm256_permute2x128_si256(y5, y6, 0x31); + y2 = _mm256_permute2x128_si256(y7, y8, 0x20); + y4 = _mm256_permute2x128_si256(y7, y8, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y4); +} + +XALIGNED(16) static const word16 L_mlkem_to_msg_avx2_hqs[] WC_X64I_UNUSED = { + 0x0680, 0x0680, 0x0680, 0x0680, 0x0680, 0x0680, 0x0680, 0x0680, + 0x0680, 0x0680, 0x0680, 0x0680, 0x0680, 0x0680, 0x0680, 0x0680, +}; + +XALIGNED(16) static const word16 L_mlkem_to_msg_avx2_hhqs[] WC_X64I_UNUSED = { + 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, + 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, 0xfcc1, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_to_msg_avx2(byte* m, sword16* p) +{ + word64 rdi, rsi, rdx, rax; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9; + + rdi = (word64)(size_t)m; + rsi = (word64)(size_t)p; + + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_to_msg_avx2_hqs, 0)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_to_msg_avx2_hhqs, 0)); + y0 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y1 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + y2 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y3 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + y4 = _mm256_srai_epi16(y0, 15); + y5 = _mm256_srai_epi16(y1, 15); + y6 = _mm256_srai_epi16(y2, 15); + y7 = _mm256_srai_epi16(y3, 15); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y5); + y2 = _mm256_xor_si256(y2, y6); + y3 = _mm256_xor_si256(y3, y7); + y0 = _mm256_add_epi16(y0, y9); + y1 = _mm256_add_epi16(y1, y9); + y2 = _mm256_add_epi16(y2, y9); + y3 = _mm256_add_epi16(y3, y9); + y0 = _mm256_packs_epi16(y0, y1); + y2 = _mm256_packs_epi16(y2, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rdx = (word32)((word32)_mm256_movemask_epi8(y0)); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + WC_S32(rdi, 0) = (word32)((word32)rdx); + WC_S32(rdi, 4) = (word32)((word32)rax); + y0 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + y1 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + y2 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + y3 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + y4 = _mm256_srai_epi16(y0, 15); + y5 = _mm256_srai_epi16(y1, 15); + y6 = _mm256_srai_epi16(y2, 15); + y7 = _mm256_srai_epi16(y3, 15); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y5); + y2 = _mm256_xor_si256(y2, y6); + y3 = _mm256_xor_si256(y3, y7); + y0 = _mm256_add_epi16(y0, y9); + y1 = _mm256_add_epi16(y1, y9); + y2 = _mm256_add_epi16(y2, y9); + y3 = _mm256_add_epi16(y3, y9); + y0 = _mm256_packs_epi16(y0, y1); + y2 = _mm256_packs_epi16(y2, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rdx = (word32)((word32)_mm256_movemask_epi8(y0)); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + WC_S32(rdi, 8) = (word32)((word32)rdx); + WC_S32(rdi, 12) = (word32)((word32)rax); + y0 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + y1 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + y2 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320))); + y3 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352))); + y4 = _mm256_srai_epi16(y0, 15); + y5 = _mm256_srai_epi16(y1, 15); + y6 = _mm256_srai_epi16(y2, 15); + y7 = _mm256_srai_epi16(y3, 15); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y5); + y2 = _mm256_xor_si256(y2, y6); + y3 = _mm256_xor_si256(y3, y7); + y0 = _mm256_add_epi16(y0, y9); + y1 = _mm256_add_epi16(y1, y9); + y2 = _mm256_add_epi16(y2, y9); + y3 = _mm256_add_epi16(y3, y9); + y0 = _mm256_packs_epi16(y0, y1); + y2 = _mm256_packs_epi16(y2, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rdx = (word32)((word32)_mm256_movemask_epi8(y0)); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + WC_S32(rdi, 16) = (word32)((word32)rdx); + WC_S32(rdi, 20) = (word32)((word32)rax); + y0 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + y1 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + y2 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448))); + y3 = _mm256_sub_epi16(y8, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480))); + y4 = _mm256_srai_epi16(y0, 15); + y5 = _mm256_srai_epi16(y1, 15); + y6 = _mm256_srai_epi16(y2, 15); + y7 = _mm256_srai_epi16(y3, 15); + y0 = _mm256_xor_si256(y0, y4); + y1 = _mm256_xor_si256(y1, y5); + y2 = _mm256_xor_si256(y2, y6); + y3 = _mm256_xor_si256(y3, y7); + y0 = _mm256_add_epi16(y0, y9); + y1 = _mm256_add_epi16(y1, y9); + y2 = _mm256_add_epi16(y2, y9); + y3 = _mm256_add_epi16(y3, y9); + y0 = _mm256_packs_epi16(y0, y1); + y2 = _mm256_packs_epi16(y2, y3); + y0 = _mm256_permute4x64_epi64(y0, 0xd8); + y2 = _mm256_permute4x64_epi64(y2, 0xd8); + rdx = (word32)((word32)_mm256_movemask_epi8(y0)); + rax = (word32)((word32)_mm256_movemask_epi8(y2)); + WC_S32(rdi, 24) = (word32)((word32)rdx); + WC_S32(rdi, 28) = (word32)((word32)rax); +} + +XALIGNED(16) static const byte L_mlkem_from_bytes_avx2_shuf[] WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x04, 0x05, 0x06, 0xff, 0x07, 0x08, 0x09, 0xff, + 0x0a, 0x0b, 0x0c, 0xff, 0x0d, 0x0e, 0x0f, 0xff, +}; + +XALIGNED(16) static const word32 L_mlkem_from_bytes_avx2_mask[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_from_bytes_avx2(sword16* p, const byte* b) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)b; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_from_bytes_avx2_shuf, + 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_from_bytes_avx2_mask, + 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y7 = _mm256_permute4x64_epi64(y5, 0xe9); + y8 = _mm256_permute4x64_epi64(y5, 0); + y6 = _mm256_permute4x64_epi64(y4, 0x3e); + y9 = _mm256_permute4x64_epi64(y4, 0x40); + y5 = _mm256_permute4x64_epi64(y3, 3); + y4 = _mm256_permute4x64_epi64(y3, 0x94); + y3 = _mm256_permute4x64_epi64(y2, 0xe9); + y10 = _mm256_permute4x64_epi64(y2, 0); + y2 = _mm256_permute4x64_epi64(y1, 0x3e); + y11 = _mm256_permute4x64_epi64(y1, 0x40); + y1 = _mm256_permute4x64_epi64(y0, 3); + y0 = _mm256_permute4x64_epi64(y0, 0x94); + y6 = _mm256_blend_epi32(y6, y8, 0xc0); + y5 = _mm256_blend_epi32(y5, y9, 0xfc); + y2 = _mm256_blend_epi32(y2, y10, 0xc0); + y1 = _mm256_blend_epi32(y1, y11, 0xfc); + y0 = _mm256_shuffle_epi8(y0, y12); + y1 = _mm256_shuffle_epi8(y1, y12); + y2 = _mm256_shuffle_epi8(y2, y12); + y3 = _mm256_shuffle_epi8(y3, y12); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y6 = _mm256_shuffle_epi8(y6, y12); + y7 = _mm256_shuffle_epi8(y7, y12); + y8 = _mm256_andnot_si256(y13, y0); + y9 = _mm256_andnot_si256(y13, y1); + y10 = _mm256_andnot_si256(y13, y2); + y11 = _mm256_andnot_si256(y13, y3); + y0 = _mm256_and_si256(y13, y0); + y1 = _mm256_and_si256(y13, y1); + y2 = _mm256_and_si256(y13, y2); + y3 = _mm256_and_si256(y13, y3); + y8 = _mm256_slli_epi32(y8, 4); + y9 = _mm256_slli_epi32(y9, 4); + y10 = _mm256_slli_epi32(y10, 4); + y11 = _mm256_slli_epi32(y11, 4); + y0 = _mm256_or_si256(y0, y8); + y1 = _mm256_or_si256(y1, y9); + y2 = _mm256_or_si256(y2, y10); + y3 = _mm256_or_si256(y3, y11); + y8 = _mm256_andnot_si256(y13, y4); + y9 = _mm256_andnot_si256(y13, y5); + y10 = _mm256_andnot_si256(y13, y6); + y11 = _mm256_andnot_si256(y13, y7); + y4 = _mm256_and_si256(y13, y4); + y5 = _mm256_and_si256(y13, y5); + y6 = _mm256_and_si256(y13, y6); + y7 = _mm256_and_si256(y13, y7); + y8 = _mm256_slli_epi32(y8, 4); + y9 = _mm256_slli_epi32(y9, 4); + y10 = _mm256_slli_epi32(y10, 4); + y11 = _mm256_slli_epi32(y11, 4); + y4 = _mm256_or_si256(y4, y8); + y5 = _mm256_or_si256(y5, y9); + y6 = _mm256_or_si256(y6, y10); + y7 = _mm256_or_si256(y7, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y7 = _mm256_permute4x64_epi64(y5, 0xe9); + y8 = _mm256_permute4x64_epi64(y5, 0); + y6 = _mm256_permute4x64_epi64(y4, 0x3e); + y9 = _mm256_permute4x64_epi64(y4, 0x40); + y5 = _mm256_permute4x64_epi64(y3, 3); + y4 = _mm256_permute4x64_epi64(y3, 0x94); + y3 = _mm256_permute4x64_epi64(y2, 0xe9); + y10 = _mm256_permute4x64_epi64(y2, 0); + y2 = _mm256_permute4x64_epi64(y1, 0x3e); + y11 = _mm256_permute4x64_epi64(y1, 0x40); + y1 = _mm256_permute4x64_epi64(y0, 3); + y0 = _mm256_permute4x64_epi64(y0, 0x94); + y6 = _mm256_blend_epi32(y6, y8, 0xc0); + y5 = _mm256_blend_epi32(y5, y9, 0xfc); + y2 = _mm256_blend_epi32(y2, y10, 0xc0); + y1 = _mm256_blend_epi32(y1, y11, 0xfc); + y0 = _mm256_shuffle_epi8(y0, y12); + y1 = _mm256_shuffle_epi8(y1, y12); + y2 = _mm256_shuffle_epi8(y2, y12); + y3 = _mm256_shuffle_epi8(y3, y12); + y4 = _mm256_shuffle_epi8(y4, y12); + y5 = _mm256_shuffle_epi8(y5, y12); + y6 = _mm256_shuffle_epi8(y6, y12); + y7 = _mm256_shuffle_epi8(y7, y12); + y8 = _mm256_andnot_si256(y13, y0); + y9 = _mm256_andnot_si256(y13, y1); + y10 = _mm256_andnot_si256(y13, y2); + y11 = _mm256_andnot_si256(y13, y3); + y0 = _mm256_and_si256(y13, y0); + y1 = _mm256_and_si256(y13, y1); + y2 = _mm256_and_si256(y13, y2); + y3 = _mm256_and_si256(y13, y3); + y8 = _mm256_slli_epi32(y8, 4); + y9 = _mm256_slli_epi32(y9, 4); + y10 = _mm256_slli_epi32(y10, 4); + y11 = _mm256_slli_epi32(y11, 4); + y0 = _mm256_or_si256(y0, y8); + y1 = _mm256_or_si256(y1, y9); + y2 = _mm256_or_si256(y2, y10); + y3 = _mm256_or_si256(y3, y11); + y8 = _mm256_andnot_si256(y13, y4); + y9 = _mm256_andnot_si256(y13, y5); + y10 = _mm256_andnot_si256(y13, y6); + y11 = _mm256_andnot_si256(y13, y7); + y4 = _mm256_and_si256(y13, y4); + y5 = _mm256_and_si256(y13, y5); + y6 = _mm256_and_si256(y13, y6); + y7 = _mm256_and_si256(y13, y7); + y8 = _mm256_slli_epi32(y8, 4); + y9 = _mm256_slli_epi32(y9, 4); + y10 = _mm256_slli_epi32(y10, 4); + y11 = _mm256_slli_epi32(y11, 4); + y4 = _mm256_or_si256(y4, y8); + y5 = _mm256_or_si256(y5, y9); + y6 = _mm256_or_si256(y6, y10); + y7 = _mm256_or_si256(y7, y11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y7); +} + +XALIGNED(16) static const word32 L_mlkem_to_bytes_avx2_mask[] WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const byte L_mlkem_to_bytes_avx2_shuf[] WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, + 0x0a, 0x0c, 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, + 0x05, 0x06, 0x08, 0x09, 0x0a, 0x0c, 0x0d, 0x0e, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x04, +}; + +XALIGNED(16) static const word32 L_mlkem_to_bytes_avx2_perm[] WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000002, 0x00000007, + 0x00000004, 0x00000005, 0x00000003, 0x00000006, +}; + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_to_bytes_avx2(byte* b, sword16* p) +{ + word64 rdi, rsi; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)b; + rsi = (word64)(size_t)p; + + y12 = _mm256_loadu_si256((const __m256i*)WC_PR(mlkem_q, 0)); + y13 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_to_bytes_avx2_mask, + 0)); + y14 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_to_bytes_avx2_shuf, + 0)); + y15 = _mm256_loadu_si256((const __m256i*)WC_PR(L_mlkem_to_bytes_avx2_perm, + 0)); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 224)); + y8 = _mm256_sub_epi16(y0, y12); + y9 = _mm256_sub_epi16(y1, y12); + y10 = _mm256_sub_epi16(y2, y12); + y11 = _mm256_sub_epi16(y3, y12); + y0 = _mm256_srai_epi16(y8, 15); + y1 = _mm256_srai_epi16(y9, 15); + y2 = _mm256_srai_epi16(y10, 15); + y3 = _mm256_srai_epi16(y11, 15); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y2 = _mm256_add_epi16(y2, y10); + y3 = _mm256_add_epi16(y3, y11); + y8 = _mm256_sub_epi16(y4, y12); + y9 = _mm256_sub_epi16(y5, y12); + y10 = _mm256_sub_epi16(y6, y12); + y11 = _mm256_sub_epi16(y7, y12); + y4 = _mm256_srai_epi16(y8, 15); + y5 = _mm256_srai_epi16(y9, 15); + y6 = _mm256_srai_epi16(y10, 15); + y7 = _mm256_srai_epi16(y11, 15); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + y6 = _mm256_and_si256(y6, y12); + y7 = _mm256_and_si256(y7, y12); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + y6 = _mm256_add_epi16(y6, y10); + y7 = _mm256_add_epi16(y7, y11); + y8 = _mm256_srli_epi32(y0, 16); + y9 = _mm256_srli_epi32(y1, 16); + y10 = _mm256_srli_epi32(y2, 16); + y11 = _mm256_srli_epi32(y3, 16); + y0 = _mm256_and_si256(y13, y0); + y1 = _mm256_and_si256(y13, y1); + y2 = _mm256_and_si256(y13, y2); + y3 = _mm256_and_si256(y13, y3); + y8 = _mm256_slli_epi32(y8, 12); + y9 = _mm256_slli_epi32(y9, 12); + y10 = _mm256_slli_epi32(y10, 12); + y11 = _mm256_slli_epi32(y11, 12); + y0 = _mm256_or_si256(y0, y8); + y1 = _mm256_or_si256(y1, y9); + y2 = _mm256_or_si256(y2, y10); + y3 = _mm256_or_si256(y3, y11); + y8 = _mm256_srli_epi32(y4, 16); + y9 = _mm256_srli_epi32(y5, 16); + y10 = _mm256_srli_epi32(y6, 16); + y11 = _mm256_srli_epi32(y7, 16); + y4 = _mm256_and_si256(y13, y4); + y5 = _mm256_and_si256(y13, y5); + y6 = _mm256_and_si256(y13, y6); + y7 = _mm256_and_si256(y13, y7); + y8 = _mm256_slli_epi32(y8, 12); + y9 = _mm256_slli_epi32(y9, 12); + y10 = _mm256_slli_epi32(y10, 12); + y11 = _mm256_slli_epi32(y11, 12); + y4 = _mm256_or_si256(y4, y8); + y5 = _mm256_or_si256(y5, y9); + y6 = _mm256_or_si256(y6, y10); + y7 = _mm256_or_si256(y7, y11); + y0 = _mm256_shuffle_epi8(y0, y14); + y1 = _mm256_shuffle_epi8(y1, y14); + y2 = _mm256_shuffle_epi8(y2, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y4 = _mm256_shuffle_epi8(y4, y14); + y5 = _mm256_shuffle_epi8(y5, y14); + y6 = _mm256_shuffle_epi8(y6, y14); + y7 = _mm256_shuffle_epi8(y7, y14); + y0 = _mm256_permutevar8x32_epi32(y0, y15); + y1 = _mm256_permutevar8x32_epi32(y1, y15); + y2 = _mm256_permutevar8x32_epi32(y2, y15); + y3 = _mm256_permutevar8x32_epi32(y3, y15); + y4 = _mm256_permutevar8x32_epi32(y4, y15); + y5 = _mm256_permutevar8x32_epi32(y5, y15); + y6 = _mm256_permutevar8x32_epi32(y6, y15); + y7 = _mm256_permutevar8x32_epi32(y7, y15); + y8 = _mm256_permute4x64_epi64(y6, 2); + y7 = _mm256_permute4x64_epi64(y7, 0x90); + y9 = _mm256_permute4x64_epi64(y5, 9); + y6 = _mm256_permute4x64_epi64(y6, 0x40); + y5 = _mm256_permute4x64_epi64(y5, 0); + y5 = _mm256_blend_epi32(y5, y4, 0x3f); + y10 = _mm256_permute4x64_epi64(y2, 2); + y4 = _mm256_permute4x64_epi64(y3, 0x90); + y11 = _mm256_permute4x64_epi64(y1, 9); + y3 = _mm256_permute4x64_epi64(y2, 0x40); + y2 = _mm256_permute4x64_epi64(y1, 0); + y2 = _mm256_blend_epi32(y2, y0, 0x3f); + y7 = _mm256_blend_epi32(y7, y8, 3); + y6 = _mm256_blend_epi32(y6, y9, 0xf); + y4 = _mm256_blend_epi32(y4, y10, 3); + y3 = _mm256_blend_epi32(y3, y11, 0xf); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rsi, 480)); + y8 = _mm256_sub_epi16(y0, y12); + y9 = _mm256_sub_epi16(y1, y12); + y10 = _mm256_sub_epi16(y2, y12); + y11 = _mm256_sub_epi16(y3, y12); + y0 = _mm256_srai_epi16(y8, 15); + y1 = _mm256_srai_epi16(y9, 15); + y2 = _mm256_srai_epi16(y10, 15); + y3 = _mm256_srai_epi16(y11, 15); + y0 = _mm256_and_si256(y0, y12); + y1 = _mm256_and_si256(y1, y12); + y2 = _mm256_and_si256(y2, y12); + y3 = _mm256_and_si256(y3, y12); + y0 = _mm256_add_epi16(y0, y8); + y1 = _mm256_add_epi16(y1, y9); + y2 = _mm256_add_epi16(y2, y10); + y3 = _mm256_add_epi16(y3, y11); + y8 = _mm256_sub_epi16(y4, y12); + y9 = _mm256_sub_epi16(y5, y12); + y10 = _mm256_sub_epi16(y6, y12); + y11 = _mm256_sub_epi16(y7, y12); + y4 = _mm256_srai_epi16(y8, 15); + y5 = _mm256_srai_epi16(y9, 15); + y6 = _mm256_srai_epi16(y10, 15); + y7 = _mm256_srai_epi16(y11, 15); + y4 = _mm256_and_si256(y4, y12); + y5 = _mm256_and_si256(y5, y12); + y6 = _mm256_and_si256(y6, y12); + y7 = _mm256_and_si256(y7, y12); + y4 = _mm256_add_epi16(y4, y8); + y5 = _mm256_add_epi16(y5, y9); + y6 = _mm256_add_epi16(y6, y10); + y7 = _mm256_add_epi16(y7, y11); + y8 = _mm256_srli_epi32(y0, 16); + y9 = _mm256_srli_epi32(y1, 16); + y10 = _mm256_srli_epi32(y2, 16); + y11 = _mm256_srli_epi32(y3, 16); + y0 = _mm256_and_si256(y13, y0); + y1 = _mm256_and_si256(y13, y1); + y2 = _mm256_and_si256(y13, y2); + y3 = _mm256_and_si256(y13, y3); + y8 = _mm256_slli_epi32(y8, 12); + y9 = _mm256_slli_epi32(y9, 12); + y10 = _mm256_slli_epi32(y10, 12); + y11 = _mm256_slli_epi32(y11, 12); + y0 = _mm256_or_si256(y0, y8); + y1 = _mm256_or_si256(y1, y9); + y2 = _mm256_or_si256(y2, y10); + y3 = _mm256_or_si256(y3, y11); + y8 = _mm256_srli_epi32(y4, 16); + y9 = _mm256_srli_epi32(y5, 16); + y10 = _mm256_srli_epi32(y6, 16); + y11 = _mm256_srli_epi32(y7, 16); + y4 = _mm256_and_si256(y13, y4); + y5 = _mm256_and_si256(y13, y5); + y6 = _mm256_and_si256(y13, y6); + y7 = _mm256_and_si256(y13, y7); + y8 = _mm256_slli_epi32(y8, 12); + y9 = _mm256_slli_epi32(y9, 12); + y10 = _mm256_slli_epi32(y10, 12); + y11 = _mm256_slli_epi32(y11, 12); + y4 = _mm256_or_si256(y4, y8); + y5 = _mm256_or_si256(y5, y9); + y6 = _mm256_or_si256(y6, y10); + y7 = _mm256_or_si256(y7, y11); + y0 = _mm256_shuffle_epi8(y0, y14); + y1 = _mm256_shuffle_epi8(y1, y14); + y2 = _mm256_shuffle_epi8(y2, y14); + y3 = _mm256_shuffle_epi8(y3, y14); + y4 = _mm256_shuffle_epi8(y4, y14); + y5 = _mm256_shuffle_epi8(y5, y14); + y6 = _mm256_shuffle_epi8(y6, y14); + y7 = _mm256_shuffle_epi8(y7, y14); + y0 = _mm256_permutevar8x32_epi32(y0, y15); + y1 = _mm256_permutevar8x32_epi32(y1, y15); + y2 = _mm256_permutevar8x32_epi32(y2, y15); + y3 = _mm256_permutevar8x32_epi32(y3, y15); + y4 = _mm256_permutevar8x32_epi32(y4, y15); + y5 = _mm256_permutevar8x32_epi32(y5, y15); + y6 = _mm256_permutevar8x32_epi32(y6, y15); + y7 = _mm256_permutevar8x32_epi32(y7, y15); + y8 = _mm256_permute4x64_epi64(y6, 2); + y7 = _mm256_permute4x64_epi64(y7, 0x90); + y9 = _mm256_permute4x64_epi64(y5, 9); + y6 = _mm256_permute4x64_epi64(y6, 0x40); + y5 = _mm256_permute4x64_epi64(y5, 0); + y5 = _mm256_blend_epi32(y5, y4, 0x3f); + y10 = _mm256_permute4x64_epi64(y2, 2); + y4 = _mm256_permute4x64_epi64(y3, 0x90); + y11 = _mm256_permute4x64_epi64(y1, 9); + y3 = _mm256_permute4x64_epi64(y2, 0x40); + y2 = _mm256_permute4x64_epi64(y1, 0); + y2 = _mm256_blend_epi32(y2, y0, 0x3f); + y7 = _mm256_blend_epi32(y7, y8, 3); + y6 = _mm256_blend_epi32(y6, y9, 0xf); + y4 = _mm256_blend_epi32(y4, y10, 3); + y3 = _mm256_blend_epi32(y3, y11, 0xf); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL int mlkem_cmp_avx2(const byte* a, const byte* b, int sz) +{ + word64 rdi, rsi, rdx, rcx, r8, rax = 0; + __m256i y0, y1, y2, y3; + word32 zf1; + word32 zf2; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + rdx = (word64)(word32)sz; + + y2 = _mm256_setzero_si256(); + y3 = _mm256_setzero_si256(); + rcx = (word32)(0); + r8 = (word32)(-1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 512))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 544))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 576))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 608))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 640)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 672)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 640))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 672))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 704)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 736)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 704))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 736))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + rdx = (word32)((word32)rdx - 0x300); + if (((word32)rdx) == (0)) { + goto L_mlkem_cmp_avx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 768)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 800)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 768))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 800))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 832)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 864)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 832))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 864))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 896)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 928)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 896))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 928))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 960)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 992)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 960))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 992))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1024)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1056)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1024))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1056))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + rdx = (word32)((word32)rdx - 0x140); + if (((word32)rdx) == (0)) { + goto L_mlkem_cmp_avx2_done; + } + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1088)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1120)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1088))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1120))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1152)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1184)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1152))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1184))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1216)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1248)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1216))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1248))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1280)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1312)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1280))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1312))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1344)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1376)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1344))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1376))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1408)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1440)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1408))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1440))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1472)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1504)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1472))); + y1 = _mm256_xor_si256(y1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1504))); + y2 = _mm256_or_si256(y2, y0); + y3 = _mm256_or_si256(y3, y1); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1536)); + y0 = _mm256_xor_si256(y0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1536))); + y2 = _mm256_or_si256(y2, y0); +L_mlkem_cmp_avx2_done: + y2 = _mm256_or_si256(y2, y3); + zf1 = _mm256_testz_si256(y2, y2); + zf2 = 1; + rax = (word32)((_mm256_testz_si256(y2, y2)) == (1) ? (word32)rcx : ( + word32)rax); + rax = (word32)((zf1) != (zf2) ? (word32)r8 : (word32)rax); + return (int)(word32)rax; +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_redistribute_21_rand_avx2(const word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, + y15; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y9 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y10 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y11 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y12 = _mm256_unpacklo_epi64(y0, y1); + y13 = _mm256_unpackhi_epi64(y0, y1); + y14 = _mm256_unpacklo_epi64(y2, y3); + y15 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y12, y14, 0x20); + y1 = _mm256_permute2x128_si256(y13, y15, 0x20); + y2 = _mm256_permute2x128_si256(y12, y14, 0x31); + y3 = _mm256_permute2x128_si256(y13, y15, 0x31); + y12 = _mm256_unpacklo_epi64(y4, y5); + y13 = _mm256_unpackhi_epi64(y4, y5); + y14 = _mm256_unpacklo_epi64(y6, y7); + y15 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y12, y14, 0x20); + y5 = _mm256_permute2x128_si256(y13, y15, 0x20); + y6 = _mm256_permute2x128_si256(y12, y14, 0x31); + y7 = _mm256_permute2x128_si256(y13, y15, 0x31); + y12 = _mm256_unpacklo_epi64(y8, y9); + y13 = _mm256_unpackhi_epi64(y8, y9); + y14 = _mm256_unpacklo_epi64(y10, y11); + y15 = _mm256_unpackhi_epi64(y10, y11); + y8 = _mm256_permute2x128_si256(y12, y14, 0x20); + y9 = _mm256_permute2x128_si256(y13, y15, 0x20); + y10 = _mm256_permute2x128_si256(y12, y14, 0x31); + y11 = _mm256_permute2x128_si256(y13, y15, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y8); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y10); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y7); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y11); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 512)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 544)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 576)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 608)); + rax = (word64)(WC_L64(rdi, 640)); + r9 = (word64)(WC_L64(rdi, 648)); + r10 = (word64)(WC_L64(rdi, 656)); + r11 = (word64)(WC_L64(rdi, 664)); + y12 = _mm256_unpacklo_epi64(y0, y1); + y13 = _mm256_unpackhi_epi64(y0, y1); + y14 = _mm256_unpacklo_epi64(y2, y3); + y15 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y12, y14, 0x20); + y1 = _mm256_permute2x128_si256(y13, y15, 0x20); + y2 = _mm256_permute2x128_si256(y12, y14, 0x31); + y3 = _mm256_permute2x128_si256(y13, y15, 0x31); + y12 = _mm256_unpacklo_epi64(y4, y5); + y13 = _mm256_unpackhi_epi64(y4, y5); + y14 = _mm256_unpacklo_epi64(y6, y7); + y15 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y12, y14, 0x20); + y5 = _mm256_permute2x128_si256(y13, y15, 0x20); + y6 = _mm256_permute2x128_si256(y12, y14, 0x31); + y7 = _mm256_permute2x128_si256(y13, y15, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), y4); + WC_S64(rsi, 160) = (word64)(rax); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), y5); + WC_S64(rdx, 160) = (word64)(r9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), y6); + WC_S64(rcx, 160) = (word64)(r10); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), y7); + WC_S64(r8, 160) = (word64)(r11); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_redistribute_17_rand_avx2(const word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y9 = _mm256_unpackhi_epi64(y0, y1); + y10 = _mm256_unpacklo_epi64(y2, y3); + y11 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y8, y10, 0x20); + y1 = _mm256_permute2x128_si256(y9, y11, 0x20); + y2 = _mm256_permute2x128_si256(y8, y10, 0x31); + y3 = _mm256_permute2x128_si256(y9, y11, 0x31); + y8 = _mm256_unpacklo_epi64(y4, y5); + y9 = _mm256_unpackhi_epi64(y4, y5); + y10 = _mm256_unpacklo_epi64(y6, y7); + y11 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y8, y10, 0x20); + y5 = _mm256_permute2x128_si256(y9, y11, 0x20); + y6 = _mm256_permute2x128_si256(y8, y10, 0x31); + y7 = _mm256_permute2x128_si256(y9, y11, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + rax = (word64)(WC_L64(rdi, 512)); + r9 = (word64)(WC_L64(rdi, 520)); + r10 = (word64)(WC_L64(rdi, 528)); + r11 = (word64)(WC_L64(rdi, 536)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y9 = _mm256_unpackhi_epi64(y0, y1); + y10 = _mm256_unpacklo_epi64(y2, y3); + y11 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y8, y10, 0x20); + y1 = _mm256_permute2x128_si256(y9, y11, 0x20); + y2 = _mm256_permute2x128_si256(y8, y10, 0x31); + y3 = _mm256_permute2x128_si256(y9, y11, 0x31); + y8 = _mm256_unpacklo_epi64(y4, y5); + y9 = _mm256_unpackhi_epi64(y4, y5); + y10 = _mm256_unpacklo_epi64(y6, y7); + y11 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y8, y10, 0x20); + y5 = _mm256_permute2x128_si256(y9, y11, 0x20); + y6 = _mm256_permute2x128_si256(y8, y10, 0x31); + y7 = _mm256_permute2x128_si256(y9, y11, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y4); + WC_S64(rsi, 128) = (word64)(rax); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y5); + WC_S64(rdx, 128) = (word64)(r9); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y6); + WC_S64(rcx, 128) = (word64)(r10); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y7); + WC_S64(r8, 128) = (word64)(r11); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_redistribute_16_rand_avx2(const word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y9 = _mm256_unpackhi_epi64(y0, y1); + y10 = _mm256_unpacklo_epi64(y2, y3); + y11 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y8, y10, 0x20); + y1 = _mm256_permute2x128_si256(y9, y11, 0x20); + y2 = _mm256_permute2x128_si256(y8, y10, 0x31); + y3 = _mm256_permute2x128_si256(y9, y11, 0x31); + y8 = _mm256_unpacklo_epi64(y4, y5); + y9 = _mm256_unpackhi_epi64(y4, y5); + y10 = _mm256_unpacklo_epi64(y6, y7); + y11 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y8, y10, 0x20); + y5 = _mm256_permute2x128_si256(y9, y11, 0x20); + y6 = _mm256_permute2x128_si256(y8, y10, 0x31); + y7 = _mm256_permute2x128_si256(y9, y11, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y7); + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 256)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 288)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 320)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 352)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 384)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 416)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 448)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 480)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y9 = _mm256_unpackhi_epi64(y0, y1); + y10 = _mm256_unpacklo_epi64(y2, y3); + y11 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y8, y10, 0x20); + y1 = _mm256_permute2x128_si256(y9, y11, 0x20); + y2 = _mm256_permute2x128_si256(y8, y10, 0x31); + y3 = _mm256_permute2x128_si256(y9, y11, 0x31); + y8 = _mm256_unpacklo_epi64(y4, y5); + y9 = _mm256_unpackhi_epi64(y4, y5); + y10 = _mm256_unpacklo_epi64(y6, y7); + y11 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y8, y10, 0x20); + y5 = _mm256_permute2x128_si256(y9, y11, 0x20); + y6 = _mm256_permute2x128_si256(y8, y10, 0x31); + y7 = _mm256_permute2x128_si256(y9, y11, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), y7); +} + +WC_X64I_TARGET("avx2") +WOLFSSL_LOCAL void mlkem_redistribute_8_rand_avx2(const word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8; + __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + y0 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 0)); + y1 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 32)); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 64)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 96)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 128)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 160)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 192)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 224)); + y8 = _mm256_unpacklo_epi64(y0, y1); + y9 = _mm256_unpackhi_epi64(y0, y1); + y10 = _mm256_unpacklo_epi64(y2, y3); + y11 = _mm256_unpackhi_epi64(y2, y3); + y0 = _mm256_permute2x128_si256(y8, y10, 0x20); + y1 = _mm256_permute2x128_si256(y9, y11, 0x20); + y2 = _mm256_permute2x128_si256(y8, y10, 0x31); + y3 = _mm256_permute2x128_si256(y9, y11, 0x31); + y8 = _mm256_unpacklo_epi64(y4, y5); + y9 = _mm256_unpackhi_epi64(y4, y5); + y10 = _mm256_unpacklo_epi64(y6, y7); + y11 = _mm256_unpackhi_epi64(y6, y7); + y4 = _mm256_permute2x128_si256(y8, y10, 0x20); + y5 = _mm256_permute2x128_si256(y9, y11, 0x20); + y6 = _mm256_permute2x128_si256(y8, y10, 0x31); + y7 = _mm256_permute2x128_si256(y9, y11, 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), y0); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), y4); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), y1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), y5); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), y2); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), y6); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), y3); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), y7); +} + +#endif /* HAVE_INTEL_AVX2 */ +#endif /* WOLFSSL_HAVE_MLKEM */ +#ifdef WOLFSSL_HAVE_MLKEM +#ifndef NO_AVX512_SUPPORT +#ifndef HAVE_INTEL_AVX512 +#define HAVE_INTEL_AVX512 +#endif /* HAVE_INTEL_AVX512 */ +#ifndef NO_AVX512_VBMI2_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI2 +#define HAVE_INTEL_AVX512_VBMI2 +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* NO_AVX512_VBMI2_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ +#ifdef HAVE_INTEL_AVX512 +XALIGNED(16) static const word16 L_mlkem_avx512_zetas[] WC_X64I_UNUSED = { + 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, + 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, 0x0a0b, + 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, + 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, 0x7b0b, + 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, + 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, 0x0b9a, + 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, + 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, 0x399a, + 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, + 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, 0x05d5, + 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, + 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, 0x34d5, + 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, + 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, 0x058e, + 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, + 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, 0xcf8e, + 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, + 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, 0x0c56, + 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, + 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, 0xae56, + 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, + 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, 0x026e, + 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, + 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, 0x6c6e, + 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, + 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, 0x0629, + 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, + 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, 0xf129, + 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, + 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, 0x00b6, + 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, + 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, 0xc2b6, + 0x023d, 0x023d, 0x023d, 0x023d, 0x023d, 0x023d, 0x023d, 0x023d, + 0x07d4, 0x07d4, 0x07d4, 0x07d4, 0x07d4, 0x07d4, 0x07d4, 0x07d4, + 0xe93d, 0xe93d, 0xe93d, 0xe93d, 0xe93d, 0xe93d, 0xe93d, 0xe93d, + 0x43d4, 0x43d4, 0x43d4, 0x43d4, 0x43d4, 0x43d4, 0x43d4, 0x43d4, + 0x0108, 0x0108, 0x0108, 0x0108, 0x0108, 0x0108, 0x0108, 0x0108, + 0x017f, 0x017f, 0x017f, 0x017f, 0x017f, 0x017f, 0x017f, 0x017f, + 0x9908, 0x9908, 0x9908, 0x9908, 0x9908, 0x9908, 0x9908, 0x9908, + 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, 0x8e7f, + 0x04c7, 0x04c7, 0x04c7, 0x04c7, 0x028c, 0x028c, 0x028c, 0x028c, + 0x0ad9, 0x0ad9, 0x0ad9, 0x0ad9, 0x03f7, 0x03f7, 0x03f7, 0x03f7, + 0xe9c7, 0xe9c7, 0xe9c7, 0xe9c7, 0xe68c, 0xe68c, 0xe68c, 0xe68c, + 0x05d9, 0x05d9, 0x05d9, 0x05d9, 0x78f7, 0x78f7, 0x78f7, 0x78f7, + 0x07f4, 0x07f4, 0x07f4, 0x07f4, 0x05d3, 0x05d3, 0x05d3, 0x05d3, + 0x0be7, 0x0be7, 0x0be7, 0x0be7, 0x06f9, 0x06f9, 0x06f9, 0x06f9, + 0xa3f4, 0xa3f4, 0xa3f4, 0xa3f4, 0x4ed3, 0x4ed3, 0x4ed3, 0x4ed3, + 0x50e7, 0x50e7, 0x50e7, 0x50e7, 0x61f9, 0x61f9, 0x61f9, 0x61f9, + 0x09c4, 0x09c4, 0x09c4, 0x09c4, 0x09c4, 0x09c4, 0x09c4, 0x09c4, + 0x05b2, 0x05b2, 0x05b2, 0x05b2, 0x05b2, 0x05b2, 0x05b2, 0x05b2, + 0x15c4, 0x15c4, 0x15c4, 0x15c4, 0x15c4, 0x15c4, 0x15c4, 0x15c4, + 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, 0xfbb2, + 0x06bf, 0x06bf, 0x06bf, 0x06bf, 0x06bf, 0x06bf, 0x06bf, 0x06bf, + 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, 0x0c7f, + 0x53bf, 0x53bf, 0x53bf, 0x53bf, 0x53bf, 0x53bf, 0x53bf, 0x53bf, + 0x997f, 0x997f, 0x997f, 0x997f, 0x997f, 0x997f, 0x997f, 0x997f, + 0x0204, 0x0204, 0x0204, 0x0204, 0x0cf9, 0x0cf9, 0x0cf9, 0x0cf9, + 0x0bc1, 0x0bc1, 0x0bc1, 0x0bc1, 0x0a67, 0x0a67, 0x0a67, 0x0a67, + 0xce04, 0xce04, 0xce04, 0xce04, 0x67f9, 0x67f9, 0x67f9, 0x67f9, + 0x3ec1, 0x3ec1, 0x3ec1, 0x3ec1, 0xcf67, 0xcf67, 0xcf67, 0xcf67, + 0x06af, 0x06af, 0x06af, 0x06af, 0x0877, 0x0877, 0x0877, 0x0877, + 0x007e, 0x007e, 0x007e, 0x007e, 0x05bd, 0x05bd, 0x05bd, 0x05bd, + 0x23af, 0x23af, 0x23af, 0x23af, 0xfd77, 0xfd77, 0xfd77, 0xfd77, + 0x9a7e, 0x9a7e, 0x9a7e, 0x9a7e, 0x6cbd, 0x6cbd, 0x6cbd, 0x6cbd, + 0x08b2, 0x08b2, 0x01ae, 0x01ae, 0x022b, 0x022b, 0x034b, 0x034b, + 0x081e, 0x081e, 0x0367, 0x0367, 0x060e, 0x060e, 0x0069, 0x0069, + 0xfeb2, 0xfeb2, 0x2bae, 0x2bae, 0xd32b, 0xd32b, 0x344b, 0x344b, + 0x821e, 0x821e, 0xc867, 0xc867, 0x500e, 0x500e, 0xab69, 0xab69, + 0x01a6, 0x01a6, 0x024b, 0x024b, 0x00b1, 0x00b1, 0x0c16, 0x0c16, + 0x0bde, 0x0bde, 0x0b35, 0x0b35, 0x0626, 0x0626, 0x0675, 0x0675, + 0x93a6, 0x93a6, 0x334b, 0x334b, 0x03b1, 0x03b1, 0xee16, 0xee16, + 0xc5de, 0xc5de, 0x5a35, 0x5a35, 0x1826, 0x1826, 0x1575, 0x1575, + 0x0c0b, 0x0c0b, 0x030a, 0x030a, 0x0487, 0x0487, 0x0c6e, 0x0c6e, + 0x09f8, 0x09f8, 0x05cb, 0x05cb, 0x0aa7, 0x0aa7, 0x045f, 0x045f, + 0x7d0b, 0x7d0b, 0x810a, 0x810a, 0x2987, 0x2987, 0x766e, 0x766e, + 0x71f8, 0x71f8, 0xb6cb, 0xb6cb, 0x8fa7, 0x8fa7, 0x315f, 0x315f, + 0x06cb, 0x06cb, 0x0284, 0x0284, 0x0999, 0x0999, 0x015d, 0x015d, + 0x01a2, 0x01a2, 0x0149, 0x0149, 0x0c65, 0x0c65, 0x0cb6, 0x0cb6, + 0xb7cb, 0xb7cb, 0x4e84, 0x4e84, 0x4499, 0x4499, 0x485d, 0x485d, + 0xc7a2, 0xc7a2, 0x4c49, 0x4c49, 0xeb65, 0xeb65, 0xceb6, 0xceb6, + 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, + 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, 0x0714, + 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, + 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, 0x0314, + 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, + 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, 0x011f, + 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, + 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, 0x6e1f, + 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, + 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, 0x00ca, + 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, + 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, 0xbeca, + 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, + 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, 0x03c2, + 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, + 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, 0x29c2, + 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, + 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, 0x084f, + 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, + 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, 0x054f, + 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, + 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, 0x073f, + 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, + 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, 0xd43f, + 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, + 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, 0x05bc, + 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, + 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, 0x79bc, + 0x0a58, 0x0a58, 0x0a58, 0x0a58, 0x0a58, 0x0a58, 0x0a58, 0x0a58, + 0x03f9, 0x03f9, 0x03f9, 0x03f9, 0x03f9, 0x03f9, 0x03f9, 0x03f9, + 0x9258, 0x9258, 0x9258, 0x9258, 0x9258, 0x9258, 0x9258, 0x9258, + 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, 0x5ef9, + 0x02dc, 0x02dc, 0x02dc, 0x02dc, 0x02dc, 0x02dc, 0x02dc, 0x02dc, + 0x0260, 0x0260, 0x0260, 0x0260, 0x0260, 0x0260, 0x0260, 0x0260, + 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, 0xd6dc, + 0x2260, 0x2260, 0x2260, 0x2260, 0x2260, 0x2260, 0x2260, 0x2260, + 0x09ac, 0x09ac, 0x09ac, 0x09ac, 0x0ca7, 0x0ca7, 0x0ca7, 0x0ca7, + 0x0bf2, 0x0bf2, 0x0bf2, 0x0bf2, 0x033e, 0x033e, 0x033e, 0x033e, + 0x4dac, 0x4dac, 0x4dac, 0x4dac, 0x91a7, 0x91a7, 0x91a7, 0x91a7, + 0xc1f2, 0xc1f2, 0xc1f2, 0xc1f2, 0xdd3e, 0xdd3e, 0xdd3e, 0xdd3e, + 0x006b, 0x006b, 0x006b, 0x006b, 0x0774, 0x0774, 0x0774, 0x0774, + 0x0c0a, 0x0c0a, 0x0c0a, 0x0c0a, 0x094a, 0x094a, 0x094a, 0x094a, + 0x916b, 0x916b, 0x916b, 0x916b, 0x2374, 0x2374, 0x2374, 0x2374, + 0x8a0a, 0x8a0a, 0x8a0a, 0x8a0a, 0x474a, 0x474a, 0x474a, 0x474a, + 0x06fb, 0x06fb, 0x06fb, 0x06fb, 0x06fb, 0x06fb, 0x06fb, 0x06fb, + 0x019b, 0x019b, 0x019b, 0x019b, 0x019b, 0x019b, 0x019b, 0x019b, + 0x47fb, 0x47fb, 0x47fb, 0x47fb, 0x47fb, 0x47fb, 0x47fb, 0x47fb, + 0x229b, 0x229b, 0x229b, 0x229b, 0x229b, 0x229b, 0x229b, 0x229b, + 0x0c34, 0x0c34, 0x0c34, 0x0c34, 0x0c34, 0x0c34, 0x0c34, 0x0c34, + 0x06de, 0x06de, 0x06de, 0x06de, 0x06de, 0x06de, 0x06de, 0x06de, + 0x6834, 0x6834, 0x6834, 0x6834, 0x6834, 0x6834, 0x6834, 0x6834, + 0xc0de, 0xc0de, 0xc0de, 0xc0de, 0xc0de, 0xc0de, 0xc0de, 0xc0de, + 0x0b73, 0x0b73, 0x0b73, 0x0b73, 0x03c1, 0x03c1, 0x03c1, 0x03c1, + 0x071d, 0x071d, 0x071d, 0x071d, 0x0a2c, 0x0a2c, 0x0a2c, 0x0a2c, + 0x3473, 0x3473, 0x3473, 0x3473, 0x36c1, 0x36c1, 0x36c1, 0x36c1, + 0x8e1d, 0x8e1d, 0x8e1d, 0x8e1d, 0xce2c, 0xce2c, 0xce2c, 0xce2c, + 0x01c0, 0x01c0, 0x01c0, 0x01c0, 0x08d8, 0x08d8, 0x08d8, 0x08d8, + 0x02a5, 0x02a5, 0x02a5, 0x02a5, 0x0806, 0x0806, 0x0806, 0x0806, + 0x41c0, 0x41c0, 0x41c0, 0x41c0, 0x10d8, 0x10d8, 0x10d8, 0x10d8, + 0xa1a5, 0xa1a5, 0xa1a5, 0xa1a5, 0xba06, 0xba06, 0xba06, 0xba06, + 0x0331, 0x0331, 0x0449, 0x0449, 0x025b, 0x025b, 0x0262, 0x0262, + 0x052a, 0x052a, 0x07fc, 0x07fc, 0x0748, 0x0748, 0x0180, 0x0180, + 0x8631, 0x8631, 0x4f49, 0x4f49, 0x635b, 0x635b, 0x0862, 0x0862, + 0xe32a, 0xe32a, 0x3bfc, 0x3bfc, 0x5f48, 0x5f48, 0x8180, 0x8180, + 0x0842, 0x0842, 0x0c79, 0x0c79, 0x04c2, 0x04c2, 0x07ca, 0x07ca, + 0x0997, 0x0997, 0x00dc, 0x00dc, 0x085e, 0x085e, 0x0686, 0x0686, + 0xae42, 0xae42, 0xe779, 0xe779, 0x2ac2, 0x2ac2, 0xc5ca, 0xc5ca, + 0x5e97, 0x5e97, 0xd4dc, 0xd4dc, 0x425e, 0x425e, 0x3886, 0x3886, + 0x0860, 0x0860, 0x0707, 0x0707, 0x0803, 0x0803, 0x031a, 0x031a, + 0x071b, 0x071b, 0x09ab, 0x09ab, 0x099b, 0x099b, 0x01de, 0x01de, + 0x2860, 0x2860, 0xac07, 0xac07, 0xe103, 0xe103, 0xb11a, 0xb11a, + 0xa81b, 0xa81b, 0x5aab, 0x5aab, 0x2a9b, 0x2a9b, 0xbbde, 0xbbde, + 0x0c95, 0x0c95, 0x0bcd, 0x0bcd, 0x03e4, 0x03e4, 0x03df, 0x03df, + 0x03be, 0x03be, 0x074d, 0x074d, 0x05f2, 0x05f2, 0x065c, 0x065c, + 0x7b95, 0x7b95, 0xa2cd, 0xa2cd, 0x6fe4, 0x6fe4, 0xb0df, 0xb0df, + 0x5dbe, 0x5dbe, 0x1e4d, 0x1e4d, 0xbbf2, 0xbbf2, 0x5a5c, 0x5a5c, +}; + +XALIGNED(32) static const word64 L_mlkem_avx512_perm20[] WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000008ULL, 0x0000000000000009ULL, + 0x0000000000000004ULL, 0x0000000000000005ULL, + 0x000000000000000cULL, 0x000000000000000dULL, +}; + +XALIGNED(32) static const word64 L_mlkem_avx512_perm31[] WC_X64I_UNUSED = { + 0x0000000000000002ULL, 0x0000000000000003ULL, + 0x000000000000000aULL, 0x000000000000000bULL, + 0x0000000000000006ULL, 0x0000000000000007ULL, + 0x000000000000000eULL, 0x000000000000000fULL, +}; + +XALIGNED(16) static const word16 L_mlkem_avx512_zetas_basemul[] + WC_X64I_UNUSED = { + 0x08b2, 0x081e, 0xf74e, 0xf7e2, 0x01ae, 0x0367, 0xfe52, 0xfc99, + 0x022b, 0x060e, 0xfdd5, 0xf9f2, 0x034b, 0x0069, 0xfcb5, 0xff97, + 0xfeb2, 0x821e, 0x014e, 0x7de2, 0x2bae, 0xc867, 0xd452, 0x3799, + 0xd32b, 0x500e, 0x2cd5, 0xaff2, 0x344b, 0xab69, 0xcbb5, 0x5497, + 0x01a6, 0x0bde, 0xfe5a, 0xf422, 0x024b, 0x0b35, 0xfdb5, 0xf4cb, + 0x00b1, 0x0626, 0xff4f, 0xf9da, 0x0c16, 0x0675, 0xf3ea, 0xf98b, + 0x93a6, 0xc5de, 0x6c5a, 0x3a22, 0x334b, 0x5a35, 0xccb5, 0xa5cb, + 0x03b1, 0x1826, 0xfc4f, 0xe7da, 0xee16, 0x1575, 0x11ea, 0xea8b, + 0x0c0b, 0x09f8, 0xf3f5, 0xf608, 0x030a, 0x05cb, 0xfcf6, 0xfa35, + 0x0487, 0x0aa7, 0xfb79, 0xf559, 0x0c6e, 0x045f, 0xf392, 0xfba1, + 0x7d0b, 0x71f8, 0x82f5, 0x8e08, 0x810a, 0xb6cb, 0x7ef6, 0x4935, + 0x2987, 0x8fa7, 0xd679, 0x7059, 0x766e, 0x315f, 0x8992, 0xcea1, + 0x06cb, 0x01a2, 0xf935, 0xfe5e, 0x0284, 0x0149, 0xfd7c, 0xfeb7, + 0x0999, 0x0c65, 0xf667, 0xf39b, 0x015d, 0x0cb6, 0xfea3, 0xf34a, + 0xb7cb, 0xc7a2, 0x4835, 0x385e, 0x4e84, 0x4c49, 0xb17c, 0xb3b7, + 0x4499, 0xeb65, 0xbb67, 0x149b, 0x485d, 0xceb6, 0xb7a3, 0x314a, + 0x0331, 0x052a, 0xfccf, 0xfad6, 0x0449, 0x07fc, 0xfbb7, 0xf804, + 0x025b, 0x0748, 0xfda5, 0xf8b8, 0x0262, 0x0180, 0xfd9e, 0xfe80, + 0x8631, 0xe32a, 0x79cf, 0x1cd6, 0x4f49, 0x3bfc, 0xb0b7, 0xc404, + 0x635b, 0x5f48, 0x9ca5, 0xa0b8, 0x0862, 0x8180, 0xf79e, 0x7e80, + 0x0842, 0x0997, 0xf7be, 0xf669, 0x0c79, 0x00dc, 0xf387, 0xff24, + 0x04c2, 0x085e, 0xfb3e, 0xf7a2, 0x07ca, 0x0686, 0xf836, 0xf97a, + 0xae42, 0x5e97, 0x51be, 0xa169, 0xe779, 0xd4dc, 0x1887, 0x2b24, + 0x2ac2, 0x425e, 0xd53e, 0xbda2, 0xc5ca, 0x3886, 0x3a36, 0xc77a, + 0x0860, 0x071b, 0xf7a0, 0xf8e5, 0x0707, 0x09ab, 0xf8f9, 0xf655, + 0x0803, 0x099b, 0xf7fd, 0xf665, 0x031a, 0x01de, 0xfce6, 0xfe22, + 0x2860, 0xa81b, 0xd7a0, 0x57e5, 0xac07, 0x5aab, 0x53f9, 0xa555, + 0xe103, 0x2a9b, 0x1efd, 0xd565, 0xb11a, 0xbbde, 0x4ee6, 0x4422, + 0x0c95, 0x03be, 0xf36b, 0xfc42, 0x0bcd, 0x074d, 0xf433, 0xf8b3, + 0x03e4, 0x05f2, 0xfc1c, 0xfa0e, 0x03df, 0x065c, 0xfc21, 0xf9a4, + 0x7b95, 0x5dbe, 0x846b, 0xa242, 0xa2cd, 0x1e4d, 0x5d33, 0xe1b3, + 0x6fe4, 0xbbf2, 0x901c, 0x440e, 0xb0df, 0x5a5c, 0x4f21, 0xa5a4, +}; + +XALIGNED(16) static const word16 L_mlkem_avx512_zetas_inv[] WC_X64I_UNUSED = { + 0x06a5, 0x06a5, 0x05b4, 0x05b4, 0x070f, 0x070f, 0x0943, 0x0943, + 0x0922, 0x0922, 0x0134, 0x0134, 0x091d, 0x091d, 0x006c, 0x006c, + 0xa5a5, 0xa5a5, 0xe1b4, 0xe1b4, 0x440f, 0x440f, 0xa243, 0xa243, + 0x4f22, 0x4f22, 0x5d34, 0x5d34, 0x901d, 0x901d, 0x846c, 0x846c, + 0x0b23, 0x0b23, 0x0356, 0x0356, 0x0366, 0x0366, 0x05e6, 0x05e6, + 0x09e7, 0x09e7, 0x05fa, 0x05fa, 0x04fe, 0x04fe, 0x04a1, 0x04a1, + 0x4423, 0x4423, 0xa556, 0xa556, 0xd566, 0xd566, 0x57e6, 0x57e6, + 0x4ee7, 0x4ee7, 0x53fa, 0x53fa, 0x1efe, 0x1efe, 0xd7a1, 0xd7a1, + 0x04fb, 0x04fb, 0x04fb, 0x04fb, 0x0a5c, 0x0a5c, 0x0a5c, 0x0a5c, + 0x0429, 0x0429, 0x0429, 0x0429, 0x0b41, 0x0b41, 0x0b41, 0x0b41, + 0x45fb, 0x45fb, 0x45fb, 0x45fb, 0x5e5c, 0x5e5c, 0x5e5c, 0x5e5c, + 0xef29, 0xef29, 0xef29, 0xef29, 0xbe41, 0xbe41, 0xbe41, 0xbe41, + 0x02d5, 0x02d5, 0x02d5, 0x02d5, 0x05e4, 0x05e4, 0x05e4, 0x05e4, + 0x0940, 0x0940, 0x0940, 0x0940, 0x018e, 0x018e, 0x018e, 0x018e, + 0x31d5, 0x31d5, 0x31d5, 0x31d5, 0x71e4, 0x71e4, 0x71e4, 0x71e4, + 0xc940, 0xc940, 0xc940, 0xc940, 0xcb8e, 0xcb8e, 0xcb8e, 0xcb8e, + 0x0623, 0x0623, 0x0623, 0x0623, 0x0623, 0x0623, 0x0623, 0x0623, + 0x00cd, 0x00cd, 0x00cd, 0x00cd, 0x00cd, 0x00cd, 0x00cd, 0x00cd, + 0x3f23, 0x3f23, 0x3f23, 0x3f23, 0x3f23, 0x3f23, 0x3f23, 0x3f23, + 0x97cd, 0x97cd, 0x97cd, 0x97cd, 0x97cd, 0x97cd, 0x97cd, 0x97cd, + 0x0b66, 0x0b66, 0x0b66, 0x0b66, 0x0b66, 0x0b66, 0x0b66, 0x0b66, + 0x0606, 0x0606, 0x0606, 0x0606, 0x0606, 0x0606, 0x0606, 0x0606, + 0xdd66, 0xdd66, 0xdd66, 0xdd66, 0xdd66, 0xdd66, 0xdd66, 0xdd66, + 0xb806, 0xb806, 0xb806, 0xb806, 0xb806, 0xb806, 0xb806, 0xb806, + 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, + 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, 0x0745, + 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, + 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, 0x8645, + 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, + 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, 0x05c2, + 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, + 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, 0x2bc2, + 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, + 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, 0x0c37, + 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, + 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, 0x4137, + 0x067b, 0x067b, 0x0c25, 0x0c25, 0x04a3, 0x04a3, 0x036a, 0x036a, + 0x0537, 0x0537, 0x0088, 0x0088, 0x083f, 0x083f, 0x04bf, 0x04bf, + 0xc77b, 0xc77b, 0x2b25, 0x2b25, 0xbda3, 0xbda3, 0xa16a, 0xa16a, + 0x3a37, 0x3a37, 0x1888, 0x1888, 0xd53f, 0xd53f, 0x51bf, 0x51bf, + 0x0b81, 0x0b81, 0x0505, 0x0505, 0x05b9, 0x05b9, 0x07d7, 0x07d7, + 0x0a9f, 0x0a9f, 0x08b8, 0x08b8, 0x0aa6, 0x0aa6, 0x09d0, 0x09d0, + 0x7e81, 0x7e81, 0xc405, 0xc405, 0xa0b9, 0xa0b9, 0x1cd7, 0x1cd7, + 0xf79f, 0xf79f, 0xb0b8, 0xb0b8, 0x9ca6, 0x9ca6, 0x79d0, 0x79d0, + 0x03b7, 0x03b7, 0x03b7, 0x03b7, 0x00f7, 0x00f7, 0x00f7, 0x00f7, + 0x058d, 0x058d, 0x058d, 0x058d, 0x0c96, 0x0c96, 0x0c96, 0x0c96, + 0xb8b7, 0xb8b7, 0xb8b7, 0xb8b7, 0x75f7, 0x75f7, 0x75f7, 0x75f7, + 0xdc8d, 0xdc8d, 0xdc8d, 0xdc8d, 0x6e96, 0x6e96, 0x6e96, 0x6e96, + 0x09c3, 0x09c3, 0x09c3, 0x09c3, 0x010f, 0x010f, 0x010f, 0x010f, + 0x005a, 0x005a, 0x005a, 0x005a, 0x0355, 0x0355, 0x0355, 0x0355, + 0x22c3, 0x22c3, 0x22c3, 0x22c3, 0x3e0f, 0x3e0f, 0x3e0f, 0x3e0f, + 0x6e5a, 0x6e5a, 0x6e5a, 0x6e5a, 0xb255, 0xb255, 0xb255, 0xb255, + 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, 0x0aa1, + 0x0a25, 0x0a25, 0x0a25, 0x0a25, 0x0a25, 0x0a25, 0x0a25, 0x0a25, + 0xdda1, 0xdda1, 0xdda1, 0xdda1, 0xdda1, 0xdda1, 0xdda1, 0xdda1, + 0x2925, 0x2925, 0x2925, 0x2925, 0x2925, 0x2925, 0x2925, 0x2925, + 0x0908, 0x0908, 0x0908, 0x0908, 0x0908, 0x0908, 0x0908, 0x0908, + 0x02a9, 0x02a9, 0x02a9, 0x02a9, 0x02a9, 0x02a9, 0x02a9, 0x02a9, + 0xa108, 0xa108, 0xa108, 0xa108, 0xa108, 0xa108, 0xa108, 0xa108, + 0x6da9, 0x6da9, 0x6da9, 0x6da9, 0x6da9, 0x6da9, 0x6da9, 0x6da9, + 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, + 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, 0x04b2, + 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, + 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, 0xfab2, + 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, + 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, 0x093f, + 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, + 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, 0xd63f, + 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, + 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, 0x0be2, + 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, + 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, 0x91e2, + 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, + 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, 0x05ed, + 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, + 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, 0xfced, + 0x004b, 0x004b, 0x0bb8, 0x0bb8, 0x009c, 0x009c, 0x0b5f, 0x0b5f, + 0x0ba4, 0x0ba4, 0x0a7d, 0x0a7d, 0x0368, 0x0368, 0x0636, 0x0636, + 0x314b, 0x314b, 0xb3b8, 0xb3b8, 0x149c, 0x149c, 0x385f, 0x385f, + 0xb7a4, 0xb7a4, 0xb17d, 0xb17d, 0xbb68, 0xbb68, 0x4836, 0x4836, + 0x08a2, 0x08a2, 0x0736, 0x0736, 0x025a, 0x025a, 0x0309, 0x0309, + 0x0093, 0x0093, 0x09f7, 0x09f7, 0x087a, 0x087a, 0x00f6, 0x00f6, + 0xcea2, 0xcea2, 0x4936, 0x4936, 0x705a, 0x705a, 0x8e09, 0x8e09, + 0x8993, 0x8993, 0x7ef7, 0x7ef7, 0xd67a, 0xd67a, 0x82f6, 0x82f6, + 0x0744, 0x0744, 0x0744, 0x0744, 0x0c83, 0x0c83, 0x0c83, 0x0c83, + 0x048a, 0x048a, 0x048a, 0x048a, 0x0652, 0x0652, 0x0652, 0x0652, + 0x9344, 0x9344, 0x9344, 0x9344, 0x6583, 0x6583, 0x6583, 0x6583, + 0x028a, 0x028a, 0x028a, 0x028a, 0xdc52, 0xdc52, 0xdc52, 0xdc52, + 0x029a, 0x029a, 0x029a, 0x029a, 0x0140, 0x0140, 0x0140, 0x0140, + 0x0008, 0x0008, 0x0008, 0x0008, 0x0afd, 0x0afd, 0x0afd, 0x0afd, + 0x309a, 0x309a, 0x309a, 0x309a, 0xc140, 0xc140, 0xc140, 0xc140, + 0x9808, 0x9808, 0x9808, 0x9808, 0x31fd, 0x31fd, 0x31fd, 0x31fd, + 0x0082, 0x0082, 0x0082, 0x0082, 0x0082, 0x0082, 0x0082, 0x0082, + 0x0642, 0x0642, 0x0642, 0x0642, 0x0642, 0x0642, 0x0642, 0x0642, + 0x6682, 0x6682, 0x6682, 0x6682, 0x6682, 0x6682, 0x6682, 0x6682, + 0xac42, 0xac42, 0xac42, 0xac42, 0xac42, 0xac42, 0xac42, 0xac42, + 0x074f, 0x074f, 0x074f, 0x074f, 0x074f, 0x074f, 0x074f, 0x074f, + 0x033d, 0x033d, 0x033d, 0x033d, 0x033d, 0x033d, 0x033d, 0x033d, + 0x044f, 0x044f, 0x044f, 0x044f, 0x044f, 0x044f, 0x044f, 0x044f, + 0xea3d, 0xea3d, 0xea3d, 0xea3d, 0xea3d, 0xea3d, 0xea3d, 0xea3d, + 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, + 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, 0x0c4b, + 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, + 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, 0x3d4b, + 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, + 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, 0x06d8, + 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, + 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, 0x0ed8, + 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, + 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, 0x0773, + 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, + 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, 0x3073, + 0x068c, 0x068c, 0x01cc, 0x01cc, 0x06db, 0x06db, 0x0123, 0x0123, + 0x00eb, 0x00eb, 0x0ab6, 0x0ab6, 0x0c50, 0x0c50, 0x0b5b, 0x0b5b, + 0xea8c, 0xea8c, 0xa5cc, 0xa5cc, 0xe7db, 0xe7db, 0x3a23, 0x3a23, + 0x11eb, 0x11eb, 0xccb6, 0xccb6, 0xfc50, 0xfc50, 0x6c5b, 0x6c5b, + 0x0c98, 0x0c98, 0x099a, 0x099a, 0x06f3, 0x06f3, 0x04e3, 0x04e3, + 0x09b6, 0x09b6, 0x0b53, 0x0b53, 0x0ad6, 0x0ad6, 0x044f, 0x044f, + 0x5498, 0x5498, 0x379a, 0x379a, 0xaff3, 0xaff3, 0x7de3, 0x7de3, + 0xcbb6, 0xcbb6, 0xd453, 0xd453, 0x2cd6, 0x2cd6, 0x014f, 0x014f, + 0x0608, 0x0608, 0x0608, 0x0608, 0x011a, 0x011a, 0x011a, 0x011a, + 0x072e, 0x072e, 0x072e, 0x072e, 0x050d, 0x050d, 0x050d, 0x050d, + 0x9e08, 0x9e08, 0x9e08, 0x9e08, 0xaf1a, 0xaf1a, 0xaf1a, 0xaf1a, + 0xb12e, 0xb12e, 0xb12e, 0xb12e, 0x5c0d, 0x5c0d, 0x5c0d, 0x5c0d, + 0x090a, 0x090a, 0x090a, 0x090a, 0x0228, 0x0228, 0x0228, 0x0228, + 0x0a75, 0x0a75, 0x0a75, 0x0a75, 0x083a, 0x083a, 0x083a, 0x083a, + 0x870a, 0x870a, 0x870a, 0x870a, 0xfa28, 0xfa28, 0xfa28, 0xfa28, + 0x1975, 0x1975, 0x1975, 0x1975, 0x163a, 0x163a, 0x163a, 0x163a, + 0x0b82, 0x0b82, 0x0b82, 0x0b82, 0x0b82, 0x0b82, 0x0b82, 0x0b82, + 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, 0x0bf9, + 0x7182, 0x7182, 0x7182, 0x7182, 0x7182, 0x7182, 0x7182, 0x7182, + 0x66f9, 0x66f9, 0x66f9, 0x66f9, 0x66f9, 0x66f9, 0x66f9, 0x66f9, + 0x052d, 0x052d, 0x052d, 0x052d, 0x052d, 0x052d, 0x052d, 0x052d, + 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, 0x0ac4, + 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, 0xbc2d, + 0x16c4, 0x16c4, 0x16c4, 0x16c4, 0x16c4, 0x16c4, 0x16c4, 0x16c4, + 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, + 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, 0x0a93, + 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, + 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, 0x9393, + 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, + 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, 0x00ab, + 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, + 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, 0x51ab, + 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, + 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, 0x072c, + 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, + 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, 0xcb2c, + 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, + 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, 0x0167, + 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, + 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, 0xc667, + 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, + 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, 0x02f6, + 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, + 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, 0x84f6, + 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, + 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, 0x05a1, + 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, + 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, 0xd8a1, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void mlkem_keygen_avx512(sword16* priv, sword16* pub, sword16* e, + const sword16* a, int kp) +{ + word64 rdi, rsi, rdx, rcx, r8, rsp, r14, r12, r9, r10, r11 = 0, r13 = 0, + rax = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), + z8 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), + z12 = _mm512_setzero_si512(), z13 = _mm512_setzero_si512(), z14, + z15, z16 = _mm512_setzero_si512(), z17, z18, + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(), + z23 = _mm512_setzero_si512(), z24 = _mm512_setzero_si512(), + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[128]; + __mmask16 k1, k2; + __mmask32 k3, k4; + + rdi = (word64)(size_t)priv; + rsi = (word64)(size_t)pub; + rdx = (word64)(size_t)e; + rcx = (word64)(size_t)a; + r8 = (word64)(word32)kp; + + rsp = (word64)(size_t)stk + 1024; + rsp = (word64)(rsp - 1024); + r14 = (word64)(0xd01); + z14 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z14 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z14)); + r14 = (word64)(0x4ebf); + z15 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z15 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z15)); + r14 = (word32)(0xaaaa); + k1 = (__mmask16)(word32)r14; + r14 = (word32)(0x5555); + k2 = (__mmask16)(word32)r14; + r14 = (word32)(0xaaaaaaaa); + k3 = (__mmask32)(word32)r14; + r14 = (word32)(0x55555555); + k4 = (__mmask32)(word32)r14; + z17 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_avx512_perm20, 0)); + z18 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_avx512_perm31, 0)); + r12 = (word64)(rdi); + r9 = (word64)((word64)(sword64)(sword32)(word32)r8); + r10 = (word64)(rdi); +L_keygen_avx512_priv: + if ((sword64)(r9) < (sword64)(2)) { + goto L_keygen_avx512_priv_odd; + } + r11 = (word64)(r10); + r11 = (word64)(r11 + 0x200); + r13 = (word64)((word64)(size_t)L_mlkem_avx512_zetas); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 480)), 1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 32))); + z8 = _mm512_mullo_epi16(z19, z12); + z9 = _mm512_mullo_epi16(z20, z12); + z19 = _mm512_mulhi_epi16(z19, z10); + z20 = _mm512_mulhi_epi16(z20, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_sub_epi16(z0, z8); + z20 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z2, z8); + z22 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z4, z8); + z24 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z6, z8); + z26 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 64))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 96))); + z8 = _mm512_mullo_epi16(z4, z12); + z9 = _mm512_mullo_epi16(z5, z12); + z4 = _mm512_mulhi_epi16(z4, z10); + z5 = _mm512_mulhi_epi16(z5, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_sub_epi16(z0, z8); + z5 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z2, z8); + z7 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1280))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1312))); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z19, z8); + z24 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z21, z8); + z26 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 160))); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1376))); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 192))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 224))); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1408))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1440))); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 352))); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1568))); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 384))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 416))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 448))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 480))); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1600))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1632))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1664))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1696))); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 512))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 544))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 576))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 608))); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z0 = _mm512_sub_epi16(z1, z0); + z2 = _mm512_sub_epi16(z3, z2); + z1 = _mm512_sub_epi16(z8, z0); + z3 = _mm512_sub_epi16(z9, z2); + z8 = _mm512_add_epi16(z8, z0); + z9 = _mm512_add_epi16(z9, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 640))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 672))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 704))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 736))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z19, z8, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1728))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z19, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1760))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z21, z9, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1792))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z21, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1824))); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z19 = _mm512_sub_epi16(z20, z19); + z21 = _mm512_sub_epi16(z22, z21); + z20 = _mm512_sub_epi16(z8, z19); + z22 = _mm512_sub_epi16(z9, z21); + z8 = _mm512_add_epi16(z8, z19); + z9 = _mm512_add_epi16(z9, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1856))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1888))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1920))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1952))); + z19 = _mm512_unpacklo_epi64(z8, z20); + z20 = _mm512_unpackhi_epi64(z8, z20); + z21 = _mm512_unpacklo_epi64(z9, z22); + z22 = _mm512_unpackhi_epi64(z9, z22); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 768))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 800))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 832))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 864))); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z4 = _mm512_sub_epi16(z5, z4); + z6 = _mm512_sub_epi16(z7, z6); + z5 = _mm512_sub_epi16(z8, z4); + z7 = _mm512_sub_epi16(z9, z6); + z8 = _mm512_add_epi16(z8, z4); + z9 = _mm512_add_epi16(z9, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 896))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 928))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 960))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 992))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z23, z8, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1984))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z23, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2016))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z25, z9, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2048))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z25, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2080))); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z23 = _mm512_sub_epi16(z24, z23); + z25 = _mm512_sub_epi16(z26, z25); + z24 = _mm512_sub_epi16(z8, z23); + z26 = _mm512_sub_epi16(z9, z25); + z8 = _mm512_add_epi16(z8, z23); + z9 = _mm512_add_epi16(z9, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2112))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2144))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2176))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2208))); + z23 = _mm512_unpacklo_epi64(z8, z24); + z24 = _mm512_unpackhi_epi64(z8, z24); + z25 = _mm512_unpacklo_epi64(z9, z26); + z26 = _mm512_unpackhi_epi64(z9, z26); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1024))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1056))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1088))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1120))); + z8 = _mm512_slli_epi64(z1, 32); + z9 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z8); + z1 = _mm512_mask_blend_epi32(k2, z1, z9); + z8 = _mm512_slli_epi64(z3, 32); + z9 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z8); + z3 = _mm512_mask_blend_epi32(k2, z3, z9); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2240))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2272))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2304))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2336))); + z8 = _mm512_slli_epi64(z20, 32); + z9 = _mm512_srli_epi64(z19, 32); + z19 = _mm512_mask_blend_epi32(k1, z19, z8); + z20 = _mm512_mask_blend_epi32(k2, z20, z9); + z8 = _mm512_slli_epi64(z22, 32); + z9 = _mm512_srli_epi64(z21, 32); + z21 = _mm512_mask_blend_epi32(k1, z21, z8); + z22 = _mm512_mask_blend_epi32(k2, z22, z9); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1184))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1216))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1248))); + z8 = _mm512_slli_epi64(z5, 32); + z9 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z8); + z5 = _mm512_mask_blend_epi32(k2, z5, z9); + z8 = _mm512_slli_epi64(z7, 32); + z9 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z8); + z7 = _mm512_mask_blend_epi32(k2, z7, z9); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2400))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2432))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2464))); + z8 = _mm512_slli_epi64(z24, 32); + z9 = _mm512_srli_epi64(z23, 32); + z23 = _mm512_mask_blend_epi32(k1, z23, z8); + z24 = _mm512_mask_blend_epi32(k2, z24, z9); + z8 = _mm512_slli_epi64(z26, 32); + z9 = _mm512_srli_epi64(z25, 32); + z25 = _mm512_mask_blend_epi32(k1, z25, z8); + z26 = _mm512_mask_blend_epi32(k2, z26, z9); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = _mm512_unpacklo_epi32(z0, z1); + z9 = _mm512_unpackhi_epi32(z0, z1); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z9); + z1 = z18; + z1 = _mm512_permutex2var_epi64(z8, z1, z9); + z8 = _mm512_unpacklo_epi32(z2, z3); + z9 = _mm512_unpackhi_epi32(z2, z3); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z8, z2, z9); + z3 = z18; + z3 = _mm512_permutex2var_epi64(z8, z3, z9); + z8 = _mm512_unpacklo_epi32(z19, z20); + z9 = _mm512_unpackhi_epi32(z19, z20); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z8, z19, z9); + z20 = z18; + z20 = _mm512_permutex2var_epi64(z8, z20, z9); + z8 = _mm512_unpacklo_epi32(z21, z22); + z9 = _mm512_unpackhi_epi32(z21, z22); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z8, z21, z9); + z22 = z18; + z22 = _mm512_permutex2var_epi64(z8, z22, z9); + z8 = _mm512_unpacklo_epi32(z4, z5); + z9 = _mm512_unpackhi_epi32(z4, z5); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z9); + z5 = z18; + z5 = _mm512_permutex2var_epi64(z8, z5, z9); + z8 = _mm512_unpacklo_epi32(z6, z7); + z9 = _mm512_unpackhi_epi32(z6, z7); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z8, z6, z9); + z7 = z18; + z7 = _mm512_permutex2var_epi64(z8, z7, z9); + z8 = _mm512_unpacklo_epi32(z23, z24); + z9 = _mm512_unpackhi_epi32(z23, z24); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z8, z23, z9); + z24 = z18; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z8 = _mm512_unpacklo_epi32(z25, z26); + z9 = _mm512_unpackhi_epi32(z25, z26); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z8, z25, z9); + z26 = z18; + z26 = _mm512_permutex2var_epi64(z8, z26, z9); + z8 = _mm512_mulhi_epi16(z0, z15); + z9 = _mm512_mulhi_epi16(z1, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z0, z8); + z9 = _mm512_sub_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 32), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z2, z15); + z9 = _mm512_mulhi_epi16(z3, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 64), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 96), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z19, z15); + z9 = _mm512_mulhi_epi16(z20, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 256), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 288), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z21, z15); + z9 = _mm512_mulhi_epi16(z22, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 320), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 352), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z4, z15); + z9 = _mm512_mulhi_epi16(z5, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 128), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 160), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z6, z15); + z9 = _mm512_mulhi_epi16(z7, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 192), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 224), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z23, z15); + z9 = _mm512_mulhi_epi16(z24, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 384), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 416), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z25, z15); + z9 = _mm512_mulhi_epi16(z26, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 448), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 480), _mm512_extracti64x4_epi64(z9, + 1)); + r10 = (word64)(r10 + 0x400); + r9 = (word64)(r9 - 2); + goto L_keygen_avx512_priv; +L_keygen_avx512_priv_odd: + if ((r9) != (1)) { + goto L_keygen_avx512_priv_done; + } + r13 = (word64)((word64)(size_t)L_mlkem_avx512_zetas); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 480)), 1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 32))); + z8 = _mm512_mullo_epi16(z19, z12); + z9 = _mm512_mullo_epi16(z20, z12); + z19 = _mm512_mulhi_epi16(z19, z10); + z20 = _mm512_mulhi_epi16(z20, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_sub_epi16(z0, z8); + z20 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z2, z8); + z22 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z4, z8); + z24 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z6, z8); + z26 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 64))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 96))); + z8 = _mm512_mullo_epi16(z4, z12); + z9 = _mm512_mullo_epi16(z5, z12); + z4 = _mm512_mulhi_epi16(z4, z10); + z5 = _mm512_mulhi_epi16(z5, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_sub_epi16(z0, z8); + z5 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z2, z8); + z7 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1280))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1312))); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z19, z8); + z24 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z21, z8); + z26 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 160))); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1376))); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 192))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 224))); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1408))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1440))); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 352))); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1568))); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 384))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 416))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 448))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 480))); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1600))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1632))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1664))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1696))); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 512))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 544))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 576))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 608))); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z0 = _mm512_sub_epi16(z1, z0); + z2 = _mm512_sub_epi16(z3, z2); + z1 = _mm512_sub_epi16(z8, z0); + z3 = _mm512_sub_epi16(z9, z2); + z8 = _mm512_add_epi16(z8, z0); + z9 = _mm512_add_epi16(z9, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 640))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 672))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 704))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 736))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z19, z8, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1728))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z19, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1760))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z21, z9, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1792))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z21, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1824))); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z19 = _mm512_sub_epi16(z20, z19); + z21 = _mm512_sub_epi16(z22, z21); + z20 = _mm512_sub_epi16(z8, z19); + z22 = _mm512_sub_epi16(z9, z21); + z8 = _mm512_add_epi16(z8, z19); + z9 = _mm512_add_epi16(z9, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1856))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1888))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1920))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1952))); + z19 = _mm512_unpacklo_epi64(z8, z20); + z20 = _mm512_unpackhi_epi64(z8, z20); + z21 = _mm512_unpacklo_epi64(z9, z22); + z22 = _mm512_unpackhi_epi64(z9, z22); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 768))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 800))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 832))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 864))); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z4 = _mm512_sub_epi16(z5, z4); + z6 = _mm512_sub_epi16(z7, z6); + z5 = _mm512_sub_epi16(z8, z4); + z7 = _mm512_sub_epi16(z9, z6); + z8 = _mm512_add_epi16(z8, z4); + z9 = _mm512_add_epi16(z9, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 896))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 928))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 960))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 992))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z23, z8, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1984))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z23, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2016))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z25, z9, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2048))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z25, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2080))); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z23 = _mm512_sub_epi16(z24, z23); + z25 = _mm512_sub_epi16(z26, z25); + z24 = _mm512_sub_epi16(z8, z23); + z26 = _mm512_sub_epi16(z9, z25); + z8 = _mm512_add_epi16(z8, z23); + z9 = _mm512_add_epi16(z9, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2112))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2144))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2176))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2208))); + z23 = _mm512_unpacklo_epi64(z8, z24); + z24 = _mm512_unpackhi_epi64(z8, z24); + z25 = _mm512_unpacklo_epi64(z9, z26); + z26 = _mm512_unpackhi_epi64(z9, z26); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1024))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1056))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1088))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1120))); + z8 = _mm512_slli_epi64(z1, 32); + z9 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z8); + z1 = _mm512_mask_blend_epi32(k2, z1, z9); + z8 = _mm512_slli_epi64(z3, 32); + z9 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z8); + z3 = _mm512_mask_blend_epi32(k2, z3, z9); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2240))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2272))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2304))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2336))); + z8 = _mm512_slli_epi64(z20, 32); + z9 = _mm512_srli_epi64(z19, 32); + z19 = _mm512_mask_blend_epi32(k1, z19, z8); + z20 = _mm512_mask_blend_epi32(k2, z20, z9); + z8 = _mm512_slli_epi64(z22, 32); + z9 = _mm512_srli_epi64(z21, 32); + z21 = _mm512_mask_blend_epi32(k1, z21, z8); + z22 = _mm512_mask_blend_epi32(k2, z22, z9); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1184))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1216))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1248))); + z8 = _mm512_slli_epi64(z5, 32); + z9 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z8); + z5 = _mm512_mask_blend_epi32(k2, z5, z9); + z8 = _mm512_slli_epi64(z7, 32); + z9 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z8); + z7 = _mm512_mask_blend_epi32(k2, z7, z9); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2400))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2432))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2464))); + z8 = _mm512_slli_epi64(z24, 32); + z9 = _mm512_srli_epi64(z23, 32); + z23 = _mm512_mask_blend_epi32(k1, z23, z8); + z24 = _mm512_mask_blend_epi32(k2, z24, z9); + z8 = _mm512_slli_epi64(z26, 32); + z9 = _mm512_srli_epi64(z25, 32); + z25 = _mm512_mask_blend_epi32(k1, z25, z8); + z26 = _mm512_mask_blend_epi32(k2, z26, z9); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = _mm512_unpacklo_epi32(z0, z1); + z9 = _mm512_unpackhi_epi32(z0, z1); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z9); + z1 = z18; + z1 = _mm512_permutex2var_epi64(z8, z1, z9); + z8 = _mm512_unpacklo_epi32(z2, z3); + z9 = _mm512_unpackhi_epi32(z2, z3); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z8, z2, z9); + z3 = z18; + z3 = _mm512_permutex2var_epi64(z8, z3, z9); + z8 = _mm512_unpacklo_epi32(z19, z20); + z9 = _mm512_unpackhi_epi32(z19, z20); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z8, z19, z9); + z20 = z18; + z20 = _mm512_permutex2var_epi64(z8, z20, z9); + z8 = _mm512_unpacklo_epi32(z21, z22); + z9 = _mm512_unpackhi_epi32(z21, z22); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z8, z21, z9); + z22 = z18; + z22 = _mm512_permutex2var_epi64(z8, z22, z9); + z8 = _mm512_unpacklo_epi32(z4, z5); + z9 = _mm512_unpackhi_epi32(z4, z5); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z9); + z5 = z18; + z5 = _mm512_permutex2var_epi64(z8, z5, z9); + z8 = _mm512_unpacklo_epi32(z6, z7); + z9 = _mm512_unpackhi_epi32(z6, z7); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z8, z6, z9); + z7 = z18; + z7 = _mm512_permutex2var_epi64(z8, z7, z9); + z8 = _mm512_unpacklo_epi32(z23, z24); + z9 = _mm512_unpackhi_epi32(z23, z24); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z8, z23, z9); + z24 = z18; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z8 = _mm512_unpacklo_epi32(z25, z26); + z9 = _mm512_unpackhi_epi32(z25, z26); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z8, z25, z9); + z26 = z18; + z26 = _mm512_permutex2var_epi64(z8, z26, z9); + z8 = _mm512_mulhi_epi16(z0, z15); + z9 = _mm512_mulhi_epi16(z1, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z0, z8); + z9 = _mm512_sub_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z2, z15); + z9 = _mm512_mulhi_epi16(z3, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z19, z15); + z9 = _mm512_mulhi_epi16(z20, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z21, z15); + z9 = _mm512_mulhi_epi16(z22, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z4, z15); + z9 = _mm512_mulhi_epi16(z5, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z6, z15); + z9 = _mm512_mulhi_epi16(z7, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z23, z15); + z9 = _mm512_mulhi_epi16(z24, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z25, z15); + z9 = _mm512_mulhi_epi16(z26, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), _mm512_extracti64x4_epi64(z9, + 1)); +L_keygen_avx512_priv_done: + r14 = (word64)(0xf301); + z13 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z13 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z13)); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + r10 = (word64)(rsi); +L_keygen_avx512_acc: + r9 = (word64)((word64)(sword64)(sword32)(word32)r8); + r13 = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + rdi = (word64)(rdi + 0x200); + r9 = (word64)(r9 - 2); + if ((r9) == (0)) { + goto L_pointwise_acc_mont_end_keygen_avx512; + } +L_pointwise_acc_mont_start_keygen_avx512: + r13 = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + rdi = (word64)(rdi + 0x200); + r9 = (word64)(r9 - 1); + if ((sword64)(r9) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_keygen_avx512; + } +L_pointwise_acc_mont_end_keygen_avx512: + r13 = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r13, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z13); + z9 = _mm512_mullo_epi16(z2, z13); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r10, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + rdi = (word64)(r12); + r10 = (word64)(r10 + 0x200); + rax = (word64)(rax - 1); + if ((sword64)(rax) > (sword64)(0)) { + goto L_keygen_avx512_acc; + } + r14 = (word64)(0x549); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z12 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z12)); + r14 = (word64)(0x5049); + z13 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r14)); + z13 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z13)); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + r10 = (word64)(rsi); +L_keygen_avx512_tomont: + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r10, 192)); + z4 = _mm512_mullo_epi16(z0, z13); + z5 = _mm512_mulhi_epi16(z0, z12); + z4 = _mm512_mulhi_epi16(z4, z14); + z0 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z1, z13); + z5 = _mm512_mulhi_epi16(z1, z12); + z4 = _mm512_mulhi_epi16(z4, z14); + z1 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z2, z13); + z5 = _mm512_mulhi_epi16(z2, z12); + z4 = _mm512_mulhi_epi16(z4, z14); + z2 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z3, z13); + z5 = _mm512_mulhi_epi16(z3, z12); + z4 = _mm512_mulhi_epi16(z4, z14); + z3 = _mm512_sub_epi16(z5, z4); + _mm512_storeu_si512((void*)WC_PW(r10, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r10, 64), z1); + _mm512_storeu_si512((void*)WC_PW(r10, 128), z2); + _mm512_storeu_si512((void*)WC_PW(r10, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r10, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r10, 448)); + z4 = _mm512_mullo_epi16(z0, z13); + z5 = _mm512_mulhi_epi16(z0, z12); + z4 = _mm512_mulhi_epi16(z4, z14); + z0 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z1, z13); + z5 = _mm512_mulhi_epi16(z1, z12); + z4 = _mm512_mulhi_epi16(z4, z14); + z1 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z2, z13); + z5 = _mm512_mulhi_epi16(z2, z12); + z4 = _mm512_mulhi_epi16(z4, z14); + z2 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z3, z13); + z5 = _mm512_mulhi_epi16(z3, z12); + z4 = _mm512_mulhi_epi16(z4, z14); + z3 = _mm512_sub_epi16(z5, z4); + _mm512_storeu_si512((void*)WC_PW(r10, 256), z0); + _mm512_storeu_si512((void*)WC_PW(r10, 320), z1); + _mm512_storeu_si512((void*)WC_PW(r10, 384), z2); + _mm512_storeu_si512((void*)WC_PW(r10, 448), z3); + r10 = (word64)(r10 + 0x200); + rax = (word64)(rax - 1); + if ((sword64)(rax) > (sword64)(0)) { + goto L_keygen_avx512_tomont; + } + rax = (word64)((word64)(sword64)(sword32)(word32)r8); +L_keygen_avx512_err: + if ((sword64)(rax) < (sword64)(2)) { + goto L_keygen_avx512_err_odd; + } + r10 = (word64)(rdx); + r10 = (word64)(r10 + 0x200); + r11 = (word64)(rsi); + r11 = (word64)(r11 + 0x200); + r13 = (word64)((word64)(size_t)L_mlkem_avx512_zetas); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 480)), 1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 32))); + z8 = _mm512_mullo_epi16(z19, z12); + z9 = _mm512_mullo_epi16(z20, z12); + z19 = _mm512_mulhi_epi16(z19, z10); + z20 = _mm512_mulhi_epi16(z20, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_sub_epi16(z0, z8); + z20 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z2, z8); + z22 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z4, z8); + z24 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z6, z8); + z26 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 64))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 96))); + z8 = _mm512_mullo_epi16(z4, z12); + z9 = _mm512_mullo_epi16(z5, z12); + z4 = _mm512_mulhi_epi16(z4, z10); + z5 = _mm512_mulhi_epi16(z5, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_sub_epi16(z0, z8); + z5 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z2, z8); + z7 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 160))); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 192))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 224))); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 352))); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 384))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 416))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 448))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 480))); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 512))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 544))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 576))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 608))); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z0 = _mm512_sub_epi16(z1, z0); + z2 = _mm512_sub_epi16(z3, z2); + z1 = _mm512_sub_epi16(z8, z0); + z3 = _mm512_sub_epi16(z9, z2); + z8 = _mm512_add_epi16(z8, z0); + z9 = _mm512_add_epi16(z9, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 640))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 672))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 704))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 736))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 768))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 800))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 832))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 864))); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z4 = _mm512_sub_epi16(z5, z4); + z6 = _mm512_sub_epi16(z7, z6); + z5 = _mm512_sub_epi16(z8, z4); + z7 = _mm512_sub_epi16(z9, z6); + z8 = _mm512_add_epi16(z8, z4); + z9 = _mm512_add_epi16(z9, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 896))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 928))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 960))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 992))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1024))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1056))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1088))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1120))); + z8 = _mm512_slli_epi64(z1, 32); + z9 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z8); + z1 = _mm512_mask_blend_epi32(k2, z1, z9); + z8 = _mm512_slli_epi64(z3, 32); + z9 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z8); + z3 = _mm512_mask_blend_epi32(k2, z3, z9); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1184))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1216))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1248))); + z8 = _mm512_slli_epi64(z5, 32); + z9 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z8); + z5 = _mm512_mask_blend_epi32(k2, z5, z9); + z8 = _mm512_slli_epi64(z7, 32); + z9 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z8); + z7 = _mm512_mask_blend_epi32(k2, z7, z9); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = _mm512_unpacklo_epi32(z0, z1); + z9 = _mm512_unpackhi_epi32(z0, z1); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z9); + z1 = z18; + z1 = _mm512_permutex2var_epi64(z8, z1, z9); + z8 = _mm512_unpacklo_epi32(z2, z3); + z9 = _mm512_unpackhi_epi32(z2, z3); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z8, z2, z9); + z3 = z18; + z3 = _mm512_permutex2var_epi64(z8, z3, z9); + z8 = _mm512_unpacklo_epi32(z4, z5); + z9 = _mm512_unpackhi_epi32(z4, z5); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z9); + z5 = z18; + z5 = _mm512_permutex2var_epi64(z8, z5, z9); + z8 = _mm512_unpacklo_epi32(z6, z7); + z9 = _mm512_unpackhi_epi32(z6, z7); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z8, z6, z9); + z7 = z18; + z7 = _mm512_permutex2var_epi64(z8, z7, z9); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mulhi_epi16(z0, z15); + z9 = _mm512_mulhi_epi16(z1, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z0, z8); + z9 = _mm512_sub_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 32), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96)), 1); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mulhi_epi16(z2, z15); + z9 = _mm512_mulhi_epi16(z3, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 64), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 96), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 128)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 160)), 1); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mulhi_epi16(z4, z15); + z9 = _mm512_mulhi_epi16(z5, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 128), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 160), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 192)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 224)), 1); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z8 = _mm512_mulhi_epi16(z6, z15); + z9 = _mm512_mulhi_epi16(z7, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 192), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 224), _mm512_extracti64x4_epi64(z9, + 1)); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1280))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1312))); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z19, z8); + z24 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z21, z8); + z26 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1376))); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1408))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1440))); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1568))); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1600))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1632))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1664))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1696))); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z19, z8, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1728))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z19, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1760))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z21, z9, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1792))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z21, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1824))); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z19 = _mm512_sub_epi16(z20, z19); + z21 = _mm512_sub_epi16(z22, z21); + z20 = _mm512_sub_epi16(z8, z19); + z22 = _mm512_sub_epi16(z9, z21); + z8 = _mm512_add_epi16(z8, z19); + z9 = _mm512_add_epi16(z9, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1856))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1888))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1920))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1952))); + z19 = _mm512_unpacklo_epi64(z8, z20); + z20 = _mm512_unpackhi_epi64(z8, z20); + z21 = _mm512_unpacklo_epi64(z9, z22); + z22 = _mm512_unpackhi_epi64(z9, z22); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z23, z8, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1984))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z23, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2016))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z25, z9, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2048))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z25, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2080))); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z23 = _mm512_sub_epi16(z24, z23); + z25 = _mm512_sub_epi16(z26, z25); + z24 = _mm512_sub_epi16(z8, z23); + z26 = _mm512_sub_epi16(z9, z25); + z8 = _mm512_add_epi16(z8, z23); + z9 = _mm512_add_epi16(z9, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2112))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2144))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2176))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2208))); + z23 = _mm512_unpacklo_epi64(z8, z24); + z24 = _mm512_unpackhi_epi64(z8, z24); + z25 = _mm512_unpacklo_epi64(z9, z26); + z26 = _mm512_unpackhi_epi64(z9, z26); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2240))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2272))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2304))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2336))); + z8 = _mm512_slli_epi64(z20, 32); + z9 = _mm512_srli_epi64(z19, 32); + z19 = _mm512_mask_blend_epi32(k1, z19, z8); + z20 = _mm512_mask_blend_epi32(k2, z20, z9); + z8 = _mm512_slli_epi64(z22, 32); + z9 = _mm512_srli_epi64(z21, 32); + z21 = _mm512_mask_blend_epi32(k1, z21, z8); + z22 = _mm512_mask_blend_epi32(k2, z22, z9); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2400))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2432))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2464))); + z8 = _mm512_slli_epi64(z24, 32); + z9 = _mm512_srli_epi64(z23, 32); + z23 = _mm512_mask_blend_epi32(k1, z23, z8); + z24 = _mm512_mask_blend_epi32(k2, z24, z9); + z8 = _mm512_slli_epi64(z26, 32); + z9 = _mm512_srli_epi64(z25, 32); + z25 = _mm512_mask_blend_epi32(k1, z25, z8); + z26 = _mm512_mask_blend_epi32(k2, z26, z9); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = _mm512_unpacklo_epi32(z19, z20); + z9 = _mm512_unpackhi_epi32(z19, z20); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z8, z19, z9); + z20 = z18; + z20 = _mm512_permutex2var_epi64(z8, z20, z9); + z8 = _mm512_unpacklo_epi32(z21, z22); + z9 = _mm512_unpackhi_epi32(z21, z22); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z8, z21, z9); + z22 = z18; + z22 = _mm512_permutex2var_epi64(z8, z22, z9); + z8 = _mm512_unpacklo_epi32(z23, z24); + z9 = _mm512_unpackhi_epi32(z23, z24); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z8, z23, z9); + z24 = z18; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z8 = _mm512_unpacklo_epi32(z25, z26); + z9 = _mm512_unpackhi_epi32(z25, z26); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z8, z25, z9); + z26 = z18; + z26 = _mm512_permutex2var_epi64(z8, z26, z9); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 256)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 288)), 1); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mulhi_epi16(z19, z15); + z9 = _mm512_mulhi_epi16(z20, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 256), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 288), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 320)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 352)), 1); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z8 = _mm512_mulhi_epi16(z21, z15); + z9 = _mm512_mulhi_epi16(z22, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 320), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 352), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 384)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 416)), 1); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z8 = _mm512_mulhi_epi16(z23, z15); + z9 = _mm512_mulhi_epi16(z24, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 384), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 416), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 448)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 480)), 1); + z25 = _mm512_add_epi16(z25, z8); + z26 = _mm512_add_epi16(z26, z9); + z8 = _mm512_mulhi_epi16(z25, z15); + z9 = _mm512_mulhi_epi16(z26, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 448), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 480), _mm512_extracti64x4_epi64(z9, + 1)); + rdx = (word64)(rdx + 0x400); + rsi = (word64)(rsi + 0x400); + rax = (word64)(rax - 2); + goto L_keygen_avx512_err; +L_keygen_avx512_err_odd: + if ((rax) != (1)) { + goto L_keygen_avx512_err_done; + } + z0 = _mm512_setzero_si512(); + _mm512_storeu_si512((void*)WC_PW(rsp, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 64), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 128), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 192), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 320), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 384), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 448), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 512), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 576), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 640), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 704), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 768), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 832), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 896), z0); + _mm512_storeu_si512((void*)WC_PW(rsp, 960), z0); + r10 = (word64)(rsp); + r11 = (word64)(r10); + r11 = (word64)(r11 + 0x200); + r13 = (word64)((word64)(size_t)L_mlkem_avx512_zetas); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 480)), 1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 32))); + z8 = _mm512_mullo_epi16(z19, z12); + z9 = _mm512_mullo_epi16(z20, z12); + z19 = _mm512_mulhi_epi16(z19, z10); + z20 = _mm512_mulhi_epi16(z20, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_sub_epi16(z0, z8); + z20 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z2, z8); + z22 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z4, z8); + z24 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z6, z8); + z26 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 64))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 96))); + z8 = _mm512_mullo_epi16(z4, z12); + z9 = _mm512_mullo_epi16(z5, z12); + z4 = _mm512_mulhi_epi16(z4, z10); + z5 = _mm512_mulhi_epi16(z5, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_sub_epi16(z0, z8); + z5 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z2, z8); + z7 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 160))); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 192))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 224))); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 352))); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 384))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 416))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 448))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 480))); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 512))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 544))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 576))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 608))); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z0 = _mm512_sub_epi16(z1, z0); + z2 = _mm512_sub_epi16(z3, z2); + z1 = _mm512_sub_epi16(z8, z0); + z3 = _mm512_sub_epi16(z9, z2); + z8 = _mm512_add_epi16(z8, z0); + z9 = _mm512_add_epi16(z9, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 640))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 672))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 704))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 736))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 768))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 800))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 832))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 864))); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z4 = _mm512_sub_epi16(z5, z4); + z6 = _mm512_sub_epi16(z7, z6); + z5 = _mm512_sub_epi16(z8, z4); + z7 = _mm512_sub_epi16(z9, z6); + z8 = _mm512_add_epi16(z8, z4); + z9 = _mm512_add_epi16(z9, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 896))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 928))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 960))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 992))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1024))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1056))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1088))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1120))); + z8 = _mm512_slli_epi64(z1, 32); + z9 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z8); + z1 = _mm512_mask_blend_epi32(k2, z1, z9); + z8 = _mm512_slli_epi64(z3, 32); + z9 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z8); + z3 = _mm512_mask_blend_epi32(k2, z3, z9); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1184))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1216))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1248))); + z8 = _mm512_slli_epi64(z5, 32); + z9 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z8); + z5 = _mm512_mask_blend_epi32(k2, z5, z9); + z8 = _mm512_slli_epi64(z7, 32); + z9 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z8); + z7 = _mm512_mask_blend_epi32(k2, z7, z9); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = _mm512_unpacklo_epi32(z0, z1); + z9 = _mm512_unpackhi_epi32(z0, z1); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z9); + z1 = z18; + z1 = _mm512_permutex2var_epi64(z8, z1, z9); + z8 = _mm512_unpacklo_epi32(z2, z3); + z9 = _mm512_unpackhi_epi32(z2, z3); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z8, z2, z9); + z3 = z18; + z3 = _mm512_permutex2var_epi64(z8, z3, z9); + z8 = _mm512_unpacklo_epi32(z4, z5); + z9 = _mm512_unpackhi_epi32(z4, z5); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z9); + z5 = z18; + z5 = _mm512_permutex2var_epi64(z8, z5, z9); + z8 = _mm512_unpacklo_epi32(z6, z7); + z9 = _mm512_unpackhi_epi32(z6, z7); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z8, z6, z9); + z7 = z18; + z7 = _mm512_permutex2var_epi64(z8, z7, z9); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mulhi_epi16(z0, z15); + z9 = _mm512_mulhi_epi16(z1, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z0, z8); + z9 = _mm512_sub_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 0), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 32), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96)), 1); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mulhi_epi16(z2, z15); + z9 = _mm512_mulhi_epi16(z3, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 64), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 96), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 128)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 160)), 1); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mulhi_epi16(z4, z15); + z9 = _mm512_mulhi_epi16(z5, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 128), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 160), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 192)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 224)), 1); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z8 = _mm512_mulhi_epi16(z6, z15); + z9 = _mm512_mulhi_epi16(z7, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 192), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 224), _mm512_extracti64x4_epi64(z9, + 1)); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1280))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1312))); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z19, z8); + z24 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z21, z8); + z26 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1376))); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1408))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1440))); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1568))); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1600))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1632))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1664))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1696))); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z19, z8, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1728))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z19, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1760))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z21, z9, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1792))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z21, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1824))); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z19 = _mm512_sub_epi16(z20, z19); + z21 = _mm512_sub_epi16(z22, z21); + z20 = _mm512_sub_epi16(z8, z19); + z22 = _mm512_sub_epi16(z9, z21); + z8 = _mm512_add_epi16(z8, z19); + z9 = _mm512_add_epi16(z9, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1856))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1888))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1920))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1952))); + z19 = _mm512_unpacklo_epi64(z8, z20); + z20 = _mm512_unpackhi_epi64(z8, z20); + z21 = _mm512_unpacklo_epi64(z9, z22); + z22 = _mm512_unpackhi_epi64(z9, z22); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z23, z8, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 1984))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z23, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2016))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z25, z9, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2048))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z25, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2080))); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z23 = _mm512_sub_epi16(z24, z23); + z25 = _mm512_sub_epi16(z26, z25); + z24 = _mm512_sub_epi16(z8, z23); + z26 = _mm512_sub_epi16(z9, z25); + z8 = _mm512_add_epi16(z8, z23); + z9 = _mm512_add_epi16(z9, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2112))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2144))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2176))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2208))); + z23 = _mm512_unpacklo_epi64(z8, z24); + z24 = _mm512_unpackhi_epi64(z8, z24); + z25 = _mm512_unpacklo_epi64(z9, z26); + z26 = _mm512_unpackhi_epi64(z9, z26); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2240))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2272))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2304))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2336))); + z8 = _mm512_slli_epi64(z20, 32); + z9 = _mm512_srli_epi64(z19, 32); + z19 = _mm512_mask_blend_epi32(k1, z19, z8); + z20 = _mm512_mask_blend_epi32(k2, z20, z9); + z8 = _mm512_slli_epi64(z22, 32); + z9 = _mm512_srli_epi64(z21, 32); + z21 = _mm512_mask_blend_epi32(k1, z21, z8); + z22 = _mm512_mask_blend_epi32(k2, z22, z9); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2400))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2432))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r13, + 2464))); + z8 = _mm512_slli_epi64(z24, 32); + z9 = _mm512_srli_epi64(z23, 32); + z23 = _mm512_mask_blend_epi32(k1, z23, z8); + z24 = _mm512_mask_blend_epi32(k2, z24, z9); + z8 = _mm512_slli_epi64(z26, 32); + z9 = _mm512_srli_epi64(z25, 32); + z25 = _mm512_mask_blend_epi32(k1, z25, z8); + z26 = _mm512_mask_blend_epi32(k2, z26, z9); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = _mm512_unpacklo_epi32(z19, z20); + z9 = _mm512_unpackhi_epi32(z19, z20); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z8, z19, z9); + z20 = z18; + z20 = _mm512_permutex2var_epi64(z8, z20, z9); + z8 = _mm512_unpacklo_epi32(z21, z22); + z9 = _mm512_unpackhi_epi32(z21, z22); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z8, z21, z9); + z22 = z18; + z22 = _mm512_permutex2var_epi64(z8, z22, z9); + z8 = _mm512_unpacklo_epi32(z23, z24); + z9 = _mm512_unpackhi_epi32(z23, z24); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z8, z23, z9); + z24 = z18; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z8 = _mm512_unpacklo_epi32(z25, z26); + z9 = _mm512_unpackhi_epi32(z25, z26); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z8, z25, z9); + z26 = z18; + z26 = _mm512_permutex2var_epi64(z8, z26, z9); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 256)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 288)), 1); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mulhi_epi16(z19, z15); + z9 = _mm512_mulhi_epi16(z20, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 256), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 288), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 320)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 352)), 1); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z8 = _mm512_mulhi_epi16(z21, z15); + z9 = _mm512_mulhi_epi16(z22, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 320), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 352), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 384)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 416)), 1); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z8 = _mm512_mulhi_epi16(z23, z15); + z9 = _mm512_mulhi_epi16(z24, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 384), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 416), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 448)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 480)), 1); + z25 = _mm512_add_epi16(z25, z8); + z26 = _mm512_add_epi16(z26, z9); + z8 = _mm512_mulhi_epi16(z25, z15); + z9 = _mm512_mulhi_epi16(z26, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 448), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r11, 480), _mm512_extracti64x4_epi64(z9, + 1)); +L_keygen_avx512_err_done: + ; +} + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void mlkem_encapsulate_avx512(const sword16* pub, sword16* bp, + sword16* vp, const sword16* at, sword16* sp, const sword16* ep, + const sword16* epp, const sword16* m, int kp) +{ + word64 rdi, rsi, rdx, rcx, r8, r9, rax, r10, r11, rsp, r13, rbx, r14, + r15 = 0, rbp = 0, r12 = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), + z8 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), + z12 = _mm512_setzero_si512(), z13 = _mm512_setzero_si512(), z14, + z15, z16 = _mm512_setzero_si512(), z17, z18, + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(), + z23 = _mm512_setzero_si512(), z24 = _mm512_setzero_si512(), + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(), + z27 = _mm512_setzero_si512(), z28 = _mm512_setzero_si512(); + XALIGNED(32) WC_X64I_SLOT stk[8]; + __mmask16 k1, k2; + __mmask32 k3, k4; + + rdi = (word64)(size_t)pub; + rsi = (word64)(size_t)bp; + rdx = (word64)(size_t)vp; + rcx = (word64)(size_t)at; + r8 = (word64)(size_t)sp; + r9 = (word64)(size_t)ep; + rax = (word64)(size_t)epp; + r10 = (word64)(size_t)m; + r11 = (word64)(word32)kp; + + rsp = (word64)(size_t)stk + 64; + rsp = (word64)(rsp - 48); + r13 = (word64)(0xd01); + z14 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r13)); + z14 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z14)); + r13 = (word64)(0x4ebf); + z15 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r13)); + z15 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z15)); + r13 = (word32)(0xaaaa); + k1 = (__mmask16)(word32)r13; + r13 = (word32)(0x5555); + k2 = (__mmask16)(word32)r13; + r13 = (word32)(0xaaaaaaaa); + k3 = (__mmask32)(word32)r13; + r13 = (word32)(0x55555555); + k4 = (__mmask32)(word32)r13; + z17 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_avx512_perm20, 0)); + z18 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_avx512_perm31, 0)); + rbx = (word64)(r8); + r13 = (word64)((word64)(sword64)(sword32)(word32)r11); + r14 = (word64)(r8); +L_encap_avx512_sp: + if ((sword64)(r13) < (sword64)(2)) { + goto L_encap_avx512_sp_odd; + } + r15 = (word64)(r14); + r15 = (word64)(r15 + 0x200); + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 480)), 1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z8 = _mm512_mullo_epi16(z19, z12); + z9 = _mm512_mullo_epi16(z20, z12); + z19 = _mm512_mulhi_epi16(z19, z10); + z20 = _mm512_mulhi_epi16(z20, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_sub_epi16(z0, z8); + z20 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z2, z8); + z22 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z4, z8); + z24 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z6, z8); + z26 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + z8 = _mm512_mullo_epi16(z4, z12); + z9 = _mm512_mullo_epi16(z5, z12); + z4 = _mm512_mulhi_epi16(z4, z10); + z5 = _mm512_mulhi_epi16(z5, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_sub_epi16(z0, z8); + z5 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z2, z8); + z7 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1280))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1312))); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z19, z8); + z24 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z21, z8); + z26 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1376))); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1408))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1440))); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352))); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1568))); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480))); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1600))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1632))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1664))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1696))); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 512))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 544))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 576))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 608))); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z0 = _mm512_sub_epi16(z1, z0); + z2 = _mm512_sub_epi16(z3, z2); + z1 = _mm512_sub_epi16(z8, z0); + z3 = _mm512_sub_epi16(z9, z2); + z8 = _mm512_add_epi16(z8, z0); + z9 = _mm512_add_epi16(z9, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 640))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 672))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 704))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 736))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z19, z8, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1728))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z19, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1760))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z21, z9, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1792))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z21, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1824))); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z19 = _mm512_sub_epi16(z20, z19); + z21 = _mm512_sub_epi16(z22, z21); + z20 = _mm512_sub_epi16(z8, z19); + z22 = _mm512_sub_epi16(z9, z21); + z8 = _mm512_add_epi16(z8, z19); + z9 = _mm512_add_epi16(z9, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1856))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1888))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1920))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1952))); + z19 = _mm512_unpacklo_epi64(z8, z20); + z20 = _mm512_unpackhi_epi64(z8, z20); + z21 = _mm512_unpacklo_epi64(z9, z22); + z22 = _mm512_unpackhi_epi64(z9, z22); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 768))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 800))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 832))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 864))); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z4 = _mm512_sub_epi16(z5, z4); + z6 = _mm512_sub_epi16(z7, z6); + z5 = _mm512_sub_epi16(z8, z4); + z7 = _mm512_sub_epi16(z9, z6); + z8 = _mm512_add_epi16(z8, z4); + z9 = _mm512_add_epi16(z9, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 896))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 928))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 960))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 992))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z23, z8, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1984))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z23, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2016))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z25, z9, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2048))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z25, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2080))); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z23 = _mm512_sub_epi16(z24, z23); + z25 = _mm512_sub_epi16(z26, z25); + z24 = _mm512_sub_epi16(z8, z23); + z26 = _mm512_sub_epi16(z9, z25); + z8 = _mm512_add_epi16(z8, z23); + z9 = _mm512_add_epi16(z9, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2112))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2144))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2176))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2208))); + z23 = _mm512_unpacklo_epi64(z8, z24); + z24 = _mm512_unpackhi_epi64(z8, z24); + z25 = _mm512_unpacklo_epi64(z9, z26); + z26 = _mm512_unpackhi_epi64(z9, z26); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1024))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1056))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1088))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1120))); + z8 = _mm512_slli_epi64(z1, 32); + z9 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z8); + z1 = _mm512_mask_blend_epi32(k2, z1, z9); + z8 = _mm512_slli_epi64(z3, 32); + z9 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z8); + z3 = _mm512_mask_blend_epi32(k2, z3, z9); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2240))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2272))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2304))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2336))); + z8 = _mm512_slli_epi64(z20, 32); + z9 = _mm512_srli_epi64(z19, 32); + z19 = _mm512_mask_blend_epi32(k1, z19, z8); + z20 = _mm512_mask_blend_epi32(k2, z20, z9); + z8 = _mm512_slli_epi64(z22, 32); + z9 = _mm512_srli_epi64(z21, 32); + z21 = _mm512_mask_blend_epi32(k1, z21, z8); + z22 = _mm512_mask_blend_epi32(k2, z22, z9); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1184))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1216))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1248))); + z8 = _mm512_slli_epi64(z5, 32); + z9 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z8); + z5 = _mm512_mask_blend_epi32(k2, z5, z9); + z8 = _mm512_slli_epi64(z7, 32); + z9 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z8); + z7 = _mm512_mask_blend_epi32(k2, z7, z9); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2400))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2432))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2464))); + z8 = _mm512_slli_epi64(z24, 32); + z9 = _mm512_srli_epi64(z23, 32); + z23 = _mm512_mask_blend_epi32(k1, z23, z8); + z24 = _mm512_mask_blend_epi32(k2, z24, z9); + z8 = _mm512_slli_epi64(z26, 32); + z9 = _mm512_srli_epi64(z25, 32); + z25 = _mm512_mask_blend_epi32(k1, z25, z8); + z26 = _mm512_mask_blend_epi32(k2, z26, z9); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = _mm512_unpacklo_epi32(z0, z1); + z9 = _mm512_unpackhi_epi32(z0, z1); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z9); + z1 = z18; + z1 = _mm512_permutex2var_epi64(z8, z1, z9); + z8 = _mm512_unpacklo_epi32(z2, z3); + z9 = _mm512_unpackhi_epi32(z2, z3); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z8, z2, z9); + z3 = z18; + z3 = _mm512_permutex2var_epi64(z8, z3, z9); + z8 = _mm512_unpacklo_epi32(z19, z20); + z9 = _mm512_unpackhi_epi32(z19, z20); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z8, z19, z9); + z20 = z18; + z20 = _mm512_permutex2var_epi64(z8, z20, z9); + z8 = _mm512_unpacklo_epi32(z21, z22); + z9 = _mm512_unpackhi_epi32(z21, z22); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z8, z21, z9); + z22 = z18; + z22 = _mm512_permutex2var_epi64(z8, z22, z9); + z8 = _mm512_unpacklo_epi32(z4, z5); + z9 = _mm512_unpackhi_epi32(z4, z5); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z9); + z5 = z18; + z5 = _mm512_permutex2var_epi64(z8, z5, z9); + z8 = _mm512_unpacklo_epi32(z6, z7); + z9 = _mm512_unpackhi_epi32(z6, z7); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z8, z6, z9); + z7 = z18; + z7 = _mm512_permutex2var_epi64(z8, z7, z9); + z8 = _mm512_unpacklo_epi32(z23, z24); + z9 = _mm512_unpackhi_epi32(z23, z24); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z8, z23, z9); + z24 = z18; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z8 = _mm512_unpacklo_epi32(z25, z26); + z9 = _mm512_unpackhi_epi32(z25, z26); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z8, z25, z9); + z26 = z18; + z26 = _mm512_permutex2var_epi64(z8, z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 0), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 32), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 64), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 64), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 96), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 96), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 256), _mm512_castsi512_si256(z19)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 256), _mm512_extracti64x4_epi64( + z19, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 288), _mm512_castsi512_si256(z20)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 288), _mm512_extracti64x4_epi64( + z20, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 320), _mm512_castsi512_si256(z21)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 320), _mm512_extracti64x4_epi64( + z21, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 352), _mm512_castsi512_si256(z22)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 352), _mm512_extracti64x4_epi64( + z22, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 128), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 128), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 160), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 160), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 192), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 192), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 224), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 224), _mm512_extracti64x4_epi64(z7, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 384), _mm512_castsi512_si256(z23)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 384), _mm512_extracti64x4_epi64( + z23, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 416), _mm512_castsi512_si256(z24)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 416), _mm512_extracti64x4_epi64( + z24, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 448), _mm512_castsi512_si256(z25)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 448), _mm512_extracti64x4_epi64( + z25, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 480), _mm512_castsi512_si256(z26)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 480), _mm512_extracti64x4_epi64( + z26, 1)); + r14 = (word64)(r14 + 0x400); + r13 = (word64)(r13 - 2); + goto L_encap_avx512_sp; +L_encap_avx512_sp_odd: + if ((r13) != (1)) { + goto L_encap_avx512_sp_done; + } + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r14, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(r14, + 480)), 1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z8 = _mm512_mullo_epi16(z19, z12); + z9 = _mm512_mullo_epi16(z20, z12); + z19 = _mm512_mulhi_epi16(z19, z10); + z20 = _mm512_mulhi_epi16(z20, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_sub_epi16(z0, z8); + z20 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z2, z8); + z22 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z4, z8); + z24 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z6, z8); + z26 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + z8 = _mm512_mullo_epi16(z4, z12); + z9 = _mm512_mullo_epi16(z5, z12); + z4 = _mm512_mulhi_epi16(z4, z10); + z5 = _mm512_mulhi_epi16(z5, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_sub_epi16(z0, z8); + z5 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z2, z8); + z7 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1280))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1312))); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z19, z8); + z24 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z21, z8); + z26 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1376))); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1408))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1440))); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352))); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1568))); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480))); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1600))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1632))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1664))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1696))); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 512))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 544))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 576))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 608))); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z0 = _mm512_sub_epi16(z1, z0); + z2 = _mm512_sub_epi16(z3, z2); + z1 = _mm512_sub_epi16(z8, z0); + z3 = _mm512_sub_epi16(z9, z2); + z8 = _mm512_add_epi16(z8, z0); + z9 = _mm512_add_epi16(z9, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 640))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 672))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 704))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 736))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z19, z8, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1728))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z19, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1760))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z21, z9, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1792))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z21, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1824))); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z19 = _mm512_sub_epi16(z20, z19); + z21 = _mm512_sub_epi16(z22, z21); + z20 = _mm512_sub_epi16(z8, z19); + z22 = _mm512_sub_epi16(z9, z21); + z8 = _mm512_add_epi16(z8, z19); + z9 = _mm512_add_epi16(z9, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1856))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1888))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1920))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1952))); + z19 = _mm512_unpacklo_epi64(z8, z20); + z20 = _mm512_unpackhi_epi64(z8, z20); + z21 = _mm512_unpacklo_epi64(z9, z22); + z22 = _mm512_unpackhi_epi64(z9, z22); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 768))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 800))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 832))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 864))); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z4 = _mm512_sub_epi16(z5, z4); + z6 = _mm512_sub_epi16(z7, z6); + z5 = _mm512_sub_epi16(z8, z4); + z7 = _mm512_sub_epi16(z9, z6); + z8 = _mm512_add_epi16(z8, z4); + z9 = _mm512_add_epi16(z9, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 896))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 928))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 960))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 992))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z23, z8, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1984))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z23, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2016))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z25, z9, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2048))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z25, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2080))); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z23 = _mm512_sub_epi16(z24, z23); + z25 = _mm512_sub_epi16(z26, z25); + z24 = _mm512_sub_epi16(z8, z23); + z26 = _mm512_sub_epi16(z9, z25); + z8 = _mm512_add_epi16(z8, z23); + z9 = _mm512_add_epi16(z9, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2112))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2144))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2176))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2208))); + z23 = _mm512_unpacklo_epi64(z8, z24); + z24 = _mm512_unpackhi_epi64(z8, z24); + z25 = _mm512_unpacklo_epi64(z9, z26); + z26 = _mm512_unpackhi_epi64(z9, z26); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1024))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1056))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1088))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1120))); + z8 = _mm512_slli_epi64(z1, 32); + z9 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z8); + z1 = _mm512_mask_blend_epi32(k2, z1, z9); + z8 = _mm512_slli_epi64(z3, 32); + z9 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z8); + z3 = _mm512_mask_blend_epi32(k2, z3, z9); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2240))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2272))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2304))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2336))); + z8 = _mm512_slli_epi64(z20, 32); + z9 = _mm512_srli_epi64(z19, 32); + z19 = _mm512_mask_blend_epi32(k1, z19, z8); + z20 = _mm512_mask_blend_epi32(k2, z20, z9); + z8 = _mm512_slli_epi64(z22, 32); + z9 = _mm512_srli_epi64(z21, 32); + z21 = _mm512_mask_blend_epi32(k1, z21, z8); + z22 = _mm512_mask_blend_epi32(k2, z22, z9); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1184))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1216))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1248))); + z8 = _mm512_slli_epi64(z5, 32); + z9 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z8); + z5 = _mm512_mask_blend_epi32(k2, z5, z9); + z8 = _mm512_slli_epi64(z7, 32); + z9 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z8); + z7 = _mm512_mask_blend_epi32(k2, z7, z9); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2400))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2432))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2464))); + z8 = _mm512_slli_epi64(z24, 32); + z9 = _mm512_srli_epi64(z23, 32); + z23 = _mm512_mask_blend_epi32(k1, z23, z8); + z24 = _mm512_mask_blend_epi32(k2, z24, z9); + z8 = _mm512_slli_epi64(z26, 32); + z9 = _mm512_srli_epi64(z25, 32); + z25 = _mm512_mask_blend_epi32(k1, z25, z8); + z26 = _mm512_mask_blend_epi32(k2, z26, z9); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = _mm512_unpacklo_epi32(z0, z1); + z9 = _mm512_unpackhi_epi32(z0, z1); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z9); + z1 = z18; + z1 = _mm512_permutex2var_epi64(z8, z1, z9); + z8 = _mm512_unpacklo_epi32(z2, z3); + z9 = _mm512_unpackhi_epi32(z2, z3); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z8, z2, z9); + z3 = z18; + z3 = _mm512_permutex2var_epi64(z8, z3, z9); + z8 = _mm512_unpacklo_epi32(z19, z20); + z9 = _mm512_unpackhi_epi32(z19, z20); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z8, z19, z9); + z20 = z18; + z20 = _mm512_permutex2var_epi64(z8, z20, z9); + z8 = _mm512_unpacklo_epi32(z21, z22); + z9 = _mm512_unpackhi_epi32(z21, z22); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z8, z21, z9); + z22 = z18; + z22 = _mm512_permutex2var_epi64(z8, z22, z9); + z8 = _mm512_unpacklo_epi32(z4, z5); + z9 = _mm512_unpackhi_epi32(z4, z5); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z9); + z5 = z18; + z5 = _mm512_permutex2var_epi64(z8, z5, z9); + z8 = _mm512_unpacklo_epi32(z6, z7); + z9 = _mm512_unpackhi_epi32(z6, z7); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z8, z6, z9); + z7 = z18; + z7 = _mm512_permutex2var_epi64(z8, z7, z9); + z8 = _mm512_unpacklo_epi32(z23, z24); + z9 = _mm512_unpackhi_epi32(z23, z24); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z8, z23, z9); + z24 = z18; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z8 = _mm512_unpacklo_epi32(z25, z26); + z9 = _mm512_unpackhi_epi32(z25, z26); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z8, z25, z9); + z26 = z18; + z26 = _mm512_permutex2var_epi64(z8, z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 0), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 32), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 64), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 64), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 96), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 96), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 256), _mm512_castsi512_si256(z19)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 256), _mm512_extracti64x4_epi64( + z19, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 288), _mm512_castsi512_si256(z20)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 288), _mm512_extracti64x4_epi64( + z20, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 320), _mm512_castsi512_si256(z21)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 320), _mm512_extracti64x4_epi64( + z21, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 352), _mm512_castsi512_si256(z22)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 352), _mm512_extracti64x4_epi64( + z22, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 128), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 128), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 160), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 160), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 192), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 192), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 224), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 224), _mm512_extracti64x4_epi64(z7, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 384), _mm512_castsi512_si256(z23)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 384), _mm512_extracti64x4_epi64( + z23, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 416), _mm512_castsi512_si256(z24)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 416), _mm512_extracti64x4_epi64( + z24, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 448), _mm512_castsi512_si256(z25)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 448), _mm512_extracti64x4_epi64( + z25, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 480), _mm512_castsi512_si256(z26)); + _mm256_storeu_si256((__m256i*)WC_PW(r14, 480), _mm512_extracti64x4_epi64( + z26, 1)); +L_encap_avx512_sp_done: + r12 = (word64)((word64)(sword64)(sword32)(word32)r11); +L_encap_avx512_calc: + if ((sword64)(r12) < (sword64)(2)) { + goto L_encap_avx512_calc_odd; + } + r13 = (word64)(0xf301); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r13)); + z12 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z12)); + r13 = (word64)((word64)(sword64)(sword32)(word32)r11); + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 2); + if ((r13) == (0)) { + goto L_pointwise_acc_mont_end_encap_bp0_avx512; + } +L_pointwise_acc_mont_start_encap_bp0_avx512: + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 1); + if ((sword64)(r13) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_encap_bp0_avx512; + } +L_pointwise_acc_mont_end_encap_bp0_avx512: + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(rbx); + r15 = (word64)(rsi); + r15 = (word64)(r15 + 0x200); + r13 = (word64)((word64)(sword64)(sword32)(word32)r11); + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 2); + if ((r13) == (0)) { + goto L_pointwise_acc_mont_end_encap_bp1_avx512; + } +L_pointwise_acc_mont_start_encap_bp1_avx512: + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 1); + if ((sword64)(r13) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_encap_bp1_avx512; + } +L_pointwise_acc_mont_end_encap_bp1_avx512: + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r15, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(rbx); + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_inv); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(r15, + 480)), 1); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z0, z9, z1); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z0 = _mm512_slli_epi64(z9, 32); + z1 = _mm512_srli_epi64(z8, 32); + z0 = _mm512_mask_blend_epi32(k1, z8, z0); + z1 = _mm512_mask_blend_epi32(k2, z9, z1); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z2, z8, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + z2 = _mm512_slli_epi64(z9, 32); + z3 = _mm512_srli_epi64(z8, 32); + z2 = _mm512_mask_blend_epi32(k1, z8, z2); + z3 = _mm512_mask_blend_epi32(k2, z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z2 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z2 = _mm512_srai_epi16(z2, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z2 = _mm512_mullo_epi16(z2, z14); + z8 = _mm512_sub_epi16(z8, z0); + z9 = _mm512_sub_epi16(z9, z2); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z19, z27, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1216))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z19, z28, z20); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1248))); + z19 = _mm512_slli_epi64(z28, 32); + z20 = _mm512_srli_epi64(z27, 32); + z19 = _mm512_mask_blend_epi32(k1, z27, z19); + z20 = _mm512_mask_blend_epi32(k2, z28, z20); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z21, z27, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1280))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z21, z28, z22); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1312))); + z21 = _mm512_slli_epi64(z28, 32); + z22 = _mm512_srli_epi64(z27, 32); + z21 = _mm512_mask_blend_epi32(k1, z27, z21); + z22 = _mm512_mask_blend_epi32(k2, z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z21 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z21 = _mm512_srai_epi16(z21, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z21 = _mm512_mullo_epi16(z21, z14); + z27 = _mm512_sub_epi16(z27, z19); + z28 = _mm512_sub_epi16(z28, z21); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + z0 = _mm512_unpacklo_epi32(z8, z1); + z1 = _mm512_unpackhi_epi32(z8, z1); + z2 = _mm512_unpacklo_epi32(z9, z3); + z3 = _mm512_unpackhi_epi32(z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1376))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1408))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1440))); + z19 = _mm512_unpacklo_epi32(z27, z20); + z20 = _mm512_unpackhi_epi32(z27, z20); + z21 = _mm512_unpacklo_epi32(z28, z22); + z22 = _mm512_unpackhi_epi32(z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z2 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z2 = _mm512_srai_epi16(z2, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z2 = _mm512_mullo_epi16(z2, z14); + z8 = _mm512_sub_epi16(z8, z0); + z9 = _mm512_sub_epi16(z9, z2); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1568))); + z19 = _mm512_unpacklo_epi64(z27, z20); + z20 = _mm512_unpackhi_epi64(z27, z20); + z21 = _mm512_unpacklo_epi64(z28, z22); + z22 = _mm512_unpackhi_epi64(z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z21 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z21 = _mm512_srai_epi16(z21, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z21 = _mm512_mullo_epi16(z21, z14); + z27 = _mm512_sub_epi16(z27, z19); + z28 = _mm512_sub_epi16(z28, z21); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z8, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z9, z2, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z9, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480))); + z8 = _mm512_sub_epi16(z0, z1); + z9 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_add_epi16(z0, z1); + z2 = _mm512_add_epi16(z2, z3); + z1 = _mm512_mullo_epi16(z8, z12); + z3 = _mm512_mullo_epi16(z9, z13); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z11); + z1 = _mm512_mulhi_epi16(z1, z14); + z3 = _mm512_mulhi_epi16(z3, z14); + z1 = _mm512_sub_epi16(z8, z1); + z3 = _mm512_sub_epi16(z9, z3); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z27, z19, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1600))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z27, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1632))); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z28, z21, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1664))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z28, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1696))); + z27 = _mm512_sub_epi16(z19, z20); + z28 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_add_epi16(z19, z20); + z21 = _mm512_add_epi16(z21, z22); + z20 = _mm512_mullo_epi16(z27, z12); + z22 = _mm512_mullo_epi16(z28, z13); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z11); + z20 = _mm512_mulhi_epi16(z20, z14); + z22 = _mm512_mulhi_epi16(z22, z14); + z20 = _mm512_sub_epi16(z27, z20); + z22 = _mm512_sub_epi16(z28, z22); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 512))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 544))); + z8 = _mm512_add_epi16(z0, z2); + z9 = _mm512_add_epi16(z1, z3); + z2 = _mm512_sub_epi16(z0, z2); + z3 = _mm512_sub_epi16(z1, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z1 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z8, z0); + z1 = _mm512_sub_epi16(z9, z1); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z2 = _mm512_sub_epi16(z2, z8); + z3 = _mm512_sub_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1728))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1760))); + z27 = _mm512_add_epi16(z19, z21); + z28 = _mm512_add_epi16(z20, z22); + z21 = _mm512_sub_epi16(z19, z21); + z22 = _mm512_sub_epi16(z20, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z20 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z20 = _mm512_srai_epi16(z20, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z20 = _mm512_mullo_epi16(z20, z14); + z19 = _mm512_sub_epi16(z27, z19); + z20 = _mm512_sub_epi16(z28, z20); + z27 = _mm512_mullo_epi16(z21, z12); + z28 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z27 = _mm512_mulhi_epi16(z27, z14); + z28 = _mm512_mulhi_epi16(z28, z14); + z21 = _mm512_sub_epi16(z21, z27); + z22 = _mm512_sub_epi16(z22, z28); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 576))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z4, z9, z5); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 608))); + z4 = _mm512_slli_epi64(z9, 32); + z5 = _mm512_srli_epi64(z8, 32); + z4 = _mm512_mask_blend_epi32(k1, z8, z4); + z5 = _mm512_mask_blend_epi32(k2, z9, z5); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z6, z8, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 640))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 672))); + z6 = _mm512_slli_epi64(z9, 32); + z7 = _mm512_srli_epi64(z8, 32); + z6 = _mm512_mask_blend_epi32(k1, z8, z6); + z7 = _mm512_mask_blend_epi32(k2, z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z6 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z6 = _mm512_srai_epi16(z6, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z6 = _mm512_mullo_epi16(z6, z14); + z8 = _mm512_sub_epi16(z8, z4); + z9 = _mm512_sub_epi16(z9, z6); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z23, z27, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1792))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z23, z28, z24); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1824))); + z23 = _mm512_slli_epi64(z28, 32); + z24 = _mm512_srli_epi64(z27, 32); + z23 = _mm512_mask_blend_epi32(k1, z27, z23); + z24 = _mm512_mask_blend_epi32(k2, z28, z24); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z25, z27, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1856))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z25, z28, z26); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1888))); + z25 = _mm512_slli_epi64(z28, 32); + z26 = _mm512_srli_epi64(z27, 32); + z25 = _mm512_mask_blend_epi32(k1, z27, z25); + z26 = _mm512_mask_blend_epi32(k2, z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z25 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z25 = _mm512_srai_epi16(z25, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z25 = _mm512_mullo_epi16(z25, z14); + z27 = _mm512_sub_epi16(z27, z23); + z28 = _mm512_sub_epi16(z28, z25); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 704))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 736))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 768))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 800))); + z4 = _mm512_unpacklo_epi32(z8, z5); + z5 = _mm512_unpackhi_epi32(z8, z5); + z6 = _mm512_unpacklo_epi32(z9, z7); + z7 = _mm512_unpackhi_epi32(z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1920))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1952))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1984))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2016))); + z23 = _mm512_unpacklo_epi32(z27, z24); + z24 = _mm512_unpackhi_epi32(z27, z24); + z25 = _mm512_unpacklo_epi32(z28, z26); + z26 = _mm512_unpackhi_epi32(z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 832))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 864))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 896))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 928))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z6 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z6 = _mm512_srai_epi16(z6, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z6 = _mm512_mullo_epi16(z6, z14); + z8 = _mm512_sub_epi16(z8, z4); + z9 = _mm512_sub_epi16(z9, z6); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2048))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2080))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2112))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2144))); + z23 = _mm512_unpacklo_epi64(z27, z24); + z24 = _mm512_unpackhi_epi64(z27, z24); + z25 = _mm512_unpacklo_epi64(z28, z26); + z26 = _mm512_unpackhi_epi64(z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z25 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z25 = _mm512_srai_epi16(z25, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z25 = _mm512_mullo_epi16(z25, z14); + z27 = _mm512_sub_epi16(z27, z23); + z28 = _mm512_sub_epi16(z28, z25); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 960))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z8, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 992))); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z9, z6, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1024))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z9, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1056))); + z8 = _mm512_sub_epi16(z4, z5); + z9 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_add_epi16(z4, z5); + z6 = _mm512_add_epi16(z6, z7); + z5 = _mm512_mullo_epi16(z8, z12); + z7 = _mm512_mullo_epi16(z9, z13); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z11); + z5 = _mm512_mulhi_epi16(z5, z14); + z7 = _mm512_mulhi_epi16(z7, z14); + z5 = _mm512_sub_epi16(z8, z5); + z7 = _mm512_sub_epi16(z9, z7); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z27, z23, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2176))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z27, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2208))); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z28, z25, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2240))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z28, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2272))); + z27 = _mm512_sub_epi16(z23, z24); + z28 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_add_epi16(z23, z24); + z25 = _mm512_add_epi16(z25, z26); + z24 = _mm512_mullo_epi16(z27, z12); + z26 = _mm512_mullo_epi16(z28, z13); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z11); + z24 = _mm512_mulhi_epi16(z24, z14); + z26 = _mm512_mulhi_epi16(z26, z14); + z24 = _mm512_sub_epi16(z27, z24); + z26 = _mm512_sub_epi16(z28, z26); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1088))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1120))); + z8 = _mm512_add_epi16(z4, z6); + z9 = _mm512_add_epi16(z5, z7); + z6 = _mm512_sub_epi16(z4, z6); + z7 = _mm512_sub_epi16(z5, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z5 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z5 = _mm512_srai_epi16(z5, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z5 = _mm512_mullo_epi16(z5, z14); + z4 = _mm512_sub_epi16(z8, z4); + z5 = _mm512_sub_epi16(z9, z5); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z6 = _mm512_sub_epi16(z6, z8); + z7 = _mm512_sub_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2304))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2336))); + z27 = _mm512_add_epi16(z23, z25); + z28 = _mm512_add_epi16(z24, z26); + z25 = _mm512_sub_epi16(z23, z25); + z26 = _mm512_sub_epi16(z24, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z24 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z24 = _mm512_srai_epi16(z24, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z24 = _mm512_mullo_epi16(z24, z14); + z23 = _mm512_sub_epi16(z27, z23); + z24 = _mm512_sub_epi16(z28, z24); + z27 = _mm512_mullo_epi16(z25, z12); + z28 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z27 = _mm512_mulhi_epi16(z27, z14); + z28 = _mm512_mulhi_epi16(z28, z14); + z25 = _mm512_sub_epi16(z25, z27); + z26 = _mm512_sub_epi16(z26, z28); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1184))); + z8 = _mm512_sub_epi16(z0, z4); + z9 = _mm512_sub_epi16(z1, z5); + z0 = _mm512_add_epi16(z0, z4); + z1 = _mm512_add_epi16(z1, z5); + z4 = _mm512_mullo_epi16(z8, z12); + z5 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z4 = _mm512_mulhi_epi16(z4, z14); + z5 = _mm512_mulhi_epi16(z5, z14); + z4 = _mm512_sub_epi16(z8, z4); + z5 = _mm512_sub_epi16(z9, z5); + z8 = _mm512_sub_epi16(z2, z6); + z9 = _mm512_sub_epi16(z3, z7); + z2 = _mm512_add_epi16(z2, z6); + z3 = _mm512_add_epi16(z3, z7); + z6 = _mm512_mullo_epi16(z8, z12); + z7 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z7 = _mm512_mulhi_epi16(z7, z14); + z6 = _mm512_sub_epi16(z8, z6); + z7 = _mm512_sub_epi16(z9, z7); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2400))); + z27 = _mm512_sub_epi16(z19, z23); + z28 = _mm512_sub_epi16(z20, z24); + z19 = _mm512_add_epi16(z19, z23); + z20 = _mm512_add_epi16(z20, z24); + z23 = _mm512_mullo_epi16(z27, z12); + z24 = _mm512_mullo_epi16(z28, z12); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z10); + z23 = _mm512_mulhi_epi16(z23, z14); + z24 = _mm512_mulhi_epi16(z24, z14); + z23 = _mm512_sub_epi16(z27, z23); + z24 = _mm512_sub_epi16(z28, z24); + z27 = _mm512_sub_epi16(z21, z25); + z28 = _mm512_sub_epi16(z22, z26); + z21 = _mm512_add_epi16(z21, z25); + z22 = _mm512_add_epi16(z22, z26); + z25 = _mm512_mullo_epi16(z27, z12); + z26 = _mm512_mullo_epi16(z28, z12); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z10); + z25 = _mm512_mulhi_epi16(z25, z14); + z26 = _mm512_mulhi_epi16(z26, z14); + z25 = _mm512_sub_epi16(z27, z25); + z26 = _mm512_sub_epi16(z28, z26); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2432))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2464))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2496))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2528))); + z8 = _mm512_sub_epi16(z0, z19); + z9 = _mm512_sub_epi16(z1, z20); + z0 = _mm512_add_epi16(z0, z19); + z1 = _mm512_add_epi16(z1, z20); + z19 = _mm512_mullo_epi16(z8, z12); + z20 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z19 = _mm512_mulhi_epi16(z19, z14); + z20 = _mm512_mulhi_epi16(z20, z14); + z19 = _mm512_sub_epi16(z8, z19); + z20 = _mm512_sub_epi16(z9, z20); + z8 = _mm512_add_epi16(z2, z21); + z9 = _mm512_add_epi16(z3, z22); + z21 = _mm512_sub_epi16(z2, z21); + z22 = _mm512_sub_epi16(z3, z22); + z2 = _mm512_mulhi_epi16(z8, z15); + z3 = _mm512_mulhi_epi16(z9, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z8, z2); + z3 = _mm512_sub_epi16(z9, z3); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z21 = _mm512_sub_epi16(z21, z8); + z22 = _mm512_sub_epi16(z22, z9); + z8 = _mm512_sub_epi16(z4, z23); + z9 = _mm512_sub_epi16(z5, z24); + z4 = _mm512_add_epi16(z4, z23); + z5 = _mm512_add_epi16(z5, z24); + z23 = _mm512_mullo_epi16(z8, z12); + z24 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z23 = _mm512_mulhi_epi16(z23, z14); + z24 = _mm512_mulhi_epi16(z24, z14); + z23 = _mm512_sub_epi16(z8, z23); + z24 = _mm512_sub_epi16(z9, z24); + z8 = _mm512_add_epi16(z6, z25); + z9 = _mm512_add_epi16(z7, z26); + z25 = _mm512_sub_epi16(z6, z25); + z26 = _mm512_sub_epi16(z7, z26); + z6 = _mm512_mulhi_epi16(z8, z15); + z7 = _mm512_mulhi_epi16(z9, z15); + z6 = _mm512_srai_epi16(z6, 10); + z7 = _mm512_srai_epi16(z7, 10); + z6 = _mm512_mullo_epi16(z6, z14); + z7 = _mm512_mullo_epi16(z7, z14); + z6 = _mm512_sub_epi16(z8, z6); + z7 = _mm512_sub_epi16(z9, z7); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z25 = _mm512_sub_epi16(z25, z8); + z26 = _mm512_sub_epi16(z26, z9); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z0 = _mm512_mulhi_epi16(z0, z11); + z1 = _mm512_mulhi_epi16(z1, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z0, z8); + z1 = _mm512_sub_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z2, z13); + z9 = _mm512_mullo_epi16(z3, z13); + z2 = _mm512_mulhi_epi16(z2, z11); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z2 = _mm512_sub_epi16(z2, z8); + z3 = _mm512_sub_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z4, z13); + z9 = _mm512_mullo_epi16(z5, z13); + z4 = _mm512_mulhi_epi16(z4, z11); + z5 = _mm512_mulhi_epi16(z5, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z4 = _mm512_sub_epi16(z4, z8); + z5 = _mm512_sub_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z6, z13); + z9 = _mm512_mullo_epi16(z7, z13); + z6 = _mm512_mulhi_epi16(z6, z11); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z6 = _mm512_sub_epi16(z6, z8); + z7 = _mm512_sub_epi16(z7, z9); + z8 = _mm512_mullo_epi16(z19, z13); + z9 = _mm512_mullo_epi16(z20, z13); + z19 = _mm512_mulhi_epi16(z19, z11); + z20 = _mm512_mulhi_epi16(z20, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z19 = _mm512_sub_epi16(z19, z8); + z20 = _mm512_sub_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z21, z13); + z9 = _mm512_mullo_epi16(z22, z13); + z21 = _mm512_mulhi_epi16(z21, z11); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z21 = _mm512_sub_epi16(z21, z8); + z22 = _mm512_sub_epi16(z22, z9); + z8 = _mm512_mullo_epi16(z23, z13); + z9 = _mm512_mullo_epi16(z24, z13); + z23 = _mm512_mulhi_epi16(z23, z11); + z24 = _mm512_mulhi_epi16(z24, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z23 = _mm512_sub_epi16(z23, z8); + z24 = _mm512_sub_epi16(z24, z9); + z8 = _mm512_mullo_epi16(z25, z13); + z9 = _mm512_mullo_epi16(z26, z13); + z25 = _mm512_mulhi_epi16(z25, z11); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z25 = _mm512_sub_epi16(z25, z8); + z26 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 0), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 32), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 64), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 96), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 128), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 160), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 192), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 224), _mm512_extracti64x4_epi64(z7, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z19)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 256), _mm512_extracti64x4_epi64( + z19, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z20)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 288), _mm512_extracti64x4_epi64( + z20, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_castsi512_si256(z21)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 320), _mm512_extracti64x4_epi64( + z21, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_castsi512_si256(z22)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 352), _mm512_extracti64x4_epi64( + z22, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z23)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 384), _mm512_extracti64x4_epi64( + z23, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z24)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 416), _mm512_extracti64x4_epi64( + z24, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_castsi512_si256(z25)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 448), _mm512_extracti64x4_epi64( + z25, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_castsi512_si256(z26)); + _mm256_storeu_si256((__m256i*)WC_PW(r15, 480), _mm512_extracti64x4_epi64( + z26, 1)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r9, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR(r9, 64)); + z6 = _mm512_loadu_si512((const void*)WC_PR(r9, 128)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r9, 192)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rsi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rsi, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r9, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(r9, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(r9, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r9, 448)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rsi, 320), z1); + _mm512_storeu_si512((void*)WC_PW(rsi, 384), z2); + _mm512_storeu_si512((void*)WC_PW(rsi, 448), z3); + r14 = (word64)(r9); + r14 = (word64)(r14 + 0x200); + z0 = _mm512_loadu_si512((const void*)WC_PR(r15, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r15, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r15, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r15, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r14, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR(r14, 64)); + z6 = _mm512_loadu_si512((const void*)WC_PR(r14, 128)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r14, 192)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(r15, 0), z0); + _mm512_storeu_si512((void*)WC_PW(r15, 64), z1); + _mm512_storeu_si512((void*)WC_PW(r15, 128), z2); + _mm512_storeu_si512((void*)WC_PW(r15, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(r15, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r15, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r15, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r15, 448)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r14, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(r14, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(r14, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r14, 448)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(r15, 256), z0); + _mm512_storeu_si512((void*)WC_PW(r15, 320), z1); + _mm512_storeu_si512((void*)WC_PW(r15, 384), z2); + _mm512_storeu_si512((void*)WC_PW(r15, 448), z3); + r9 = (word64)(r9 + 0x400); + rsi = (word64)(rsi + 0x400); + r12 = (word64)(r12 - 2); + goto L_encap_avx512_calc; +L_encap_avx512_calc_odd: + r13 = (word64)(0xf301); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r13)); + z12 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z12)); + r13 = (word64)((word64)(sword64)(sword32)(word32)r11); + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rdi = (word64)(rdi + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 2); + if ((r13) == (0)) { + goto L_pointwise_acc_mont_end_encap_v_avx512; + } +L_pointwise_acc_mont_start_encap_v_avx512: + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rdi = (word64)(rdi + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 1); + if ((sword64)(r13) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_encap_v_avx512; + } +L_pointwise_acc_mont_end_encap_v_avx512: + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rdi = (word64)(rdi + 0x200); + r8 = (word64)(rbx); + if ((r12) != (1)) { + goto L_encap_avx512_vp_single; + } + r13 = (word64)((word64)(sword64)(sword32)(word32)r11); + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 2); + if ((r13) == (0)) { + goto L_pointwise_acc_mont_end_encap_bp_odd_avx512; + } +L_pointwise_acc_mont_start_encap_bp_odd_avx512: + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(r8 + 0x200); + r13 = (word64)(r13 - 1); + if ((sword64)(r13) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_encap_bp_odd_avx512; + } +L_pointwise_acc_mont_end_encap_bp_odd_avx512: + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rcx, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r8, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r8, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rcx = (word64)(rcx + 0x200); + r8 = (word64)(rbx); + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_inv); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z0, z9, z1); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z0 = _mm512_slli_epi64(z9, 32); + z1 = _mm512_srli_epi64(z8, 32); + z0 = _mm512_mask_blend_epi32(k1, z8, z0); + z1 = _mm512_mask_blend_epi32(k2, z9, z1); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z2, z8, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + z2 = _mm512_slli_epi64(z9, 32); + z3 = _mm512_srli_epi64(z8, 32); + z2 = _mm512_mask_blend_epi32(k1, z8, z2); + z3 = _mm512_mask_blend_epi32(k2, z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z2 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z2 = _mm512_srai_epi16(z2, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z2 = _mm512_mullo_epi16(z2, z14); + z8 = _mm512_sub_epi16(z8, z0); + z9 = _mm512_sub_epi16(z9, z2); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z19, z27, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1216))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z19, z28, z20); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1248))); + z19 = _mm512_slli_epi64(z28, 32); + z20 = _mm512_srli_epi64(z27, 32); + z19 = _mm512_mask_blend_epi32(k1, z27, z19); + z20 = _mm512_mask_blend_epi32(k2, z28, z20); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z21, z27, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1280))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z21, z28, z22); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1312))); + z21 = _mm512_slli_epi64(z28, 32); + z22 = _mm512_srli_epi64(z27, 32); + z21 = _mm512_mask_blend_epi32(k1, z27, z21); + z22 = _mm512_mask_blend_epi32(k2, z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z21 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z21 = _mm512_srai_epi16(z21, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z21 = _mm512_mullo_epi16(z21, z14); + z27 = _mm512_sub_epi16(z27, z19); + z28 = _mm512_sub_epi16(z28, z21); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + z0 = _mm512_unpacklo_epi32(z8, z1); + z1 = _mm512_unpackhi_epi32(z8, z1); + z2 = _mm512_unpacklo_epi32(z9, z3); + z3 = _mm512_unpackhi_epi32(z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1376))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1408))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1440))); + z19 = _mm512_unpacklo_epi32(z27, z20); + z20 = _mm512_unpackhi_epi32(z27, z20); + z21 = _mm512_unpacklo_epi32(z28, z22); + z22 = _mm512_unpackhi_epi32(z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z2 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z2 = _mm512_srai_epi16(z2, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z2 = _mm512_mullo_epi16(z2, z14); + z8 = _mm512_sub_epi16(z8, z0); + z9 = _mm512_sub_epi16(z9, z2); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1568))); + z19 = _mm512_unpacklo_epi64(z27, z20); + z20 = _mm512_unpackhi_epi64(z27, z20); + z21 = _mm512_unpacklo_epi64(z28, z22); + z22 = _mm512_unpackhi_epi64(z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z21 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z21 = _mm512_srai_epi16(z21, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z21 = _mm512_mullo_epi16(z21, z14); + z27 = _mm512_sub_epi16(z27, z19); + z28 = _mm512_sub_epi16(z28, z21); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z8, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z9, z2, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z9, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480))); + z8 = _mm512_sub_epi16(z0, z1); + z9 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_add_epi16(z0, z1); + z2 = _mm512_add_epi16(z2, z3); + z1 = _mm512_mullo_epi16(z8, z12); + z3 = _mm512_mullo_epi16(z9, z13); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z11); + z1 = _mm512_mulhi_epi16(z1, z14); + z3 = _mm512_mulhi_epi16(z3, z14); + z1 = _mm512_sub_epi16(z8, z1); + z3 = _mm512_sub_epi16(z9, z3); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z27, z19, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1600))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z27, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1632))); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z28, z21, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1664))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z28, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1696))); + z27 = _mm512_sub_epi16(z19, z20); + z28 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_add_epi16(z19, z20); + z21 = _mm512_add_epi16(z21, z22); + z20 = _mm512_mullo_epi16(z27, z12); + z22 = _mm512_mullo_epi16(z28, z13); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z11); + z20 = _mm512_mulhi_epi16(z20, z14); + z22 = _mm512_mulhi_epi16(z22, z14); + z20 = _mm512_sub_epi16(z27, z20); + z22 = _mm512_sub_epi16(z28, z22); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 512))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 544))); + z8 = _mm512_add_epi16(z0, z2); + z9 = _mm512_add_epi16(z1, z3); + z2 = _mm512_sub_epi16(z0, z2); + z3 = _mm512_sub_epi16(z1, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z1 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z8, z0); + z1 = _mm512_sub_epi16(z9, z1); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z2 = _mm512_sub_epi16(z2, z8); + z3 = _mm512_sub_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1728))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1760))); + z27 = _mm512_add_epi16(z19, z21); + z28 = _mm512_add_epi16(z20, z22); + z21 = _mm512_sub_epi16(z19, z21); + z22 = _mm512_sub_epi16(z20, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z20 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z20 = _mm512_srai_epi16(z20, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z20 = _mm512_mullo_epi16(z20, z14); + z19 = _mm512_sub_epi16(z27, z19); + z20 = _mm512_sub_epi16(z28, z20); + z27 = _mm512_mullo_epi16(z21, z12); + z28 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z27 = _mm512_mulhi_epi16(z27, z14); + z28 = _mm512_mulhi_epi16(z28, z14); + z21 = _mm512_sub_epi16(z21, z27); + z22 = _mm512_sub_epi16(z22, z28); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 576))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z4, z9, z5); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 608))); + z4 = _mm512_slli_epi64(z9, 32); + z5 = _mm512_srli_epi64(z8, 32); + z4 = _mm512_mask_blend_epi32(k1, z8, z4); + z5 = _mm512_mask_blend_epi32(k2, z9, z5); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z6, z8, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 640))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 672))); + z6 = _mm512_slli_epi64(z9, 32); + z7 = _mm512_srli_epi64(z8, 32); + z6 = _mm512_mask_blend_epi32(k1, z8, z6); + z7 = _mm512_mask_blend_epi32(k2, z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z6 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z6 = _mm512_srai_epi16(z6, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z6 = _mm512_mullo_epi16(z6, z14); + z8 = _mm512_sub_epi16(z8, z4); + z9 = _mm512_sub_epi16(z9, z6); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z23, z27, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1792))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z23, z28, z24); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1824))); + z23 = _mm512_slli_epi64(z28, 32); + z24 = _mm512_srli_epi64(z27, 32); + z23 = _mm512_mask_blend_epi32(k1, z27, z23); + z24 = _mm512_mask_blend_epi32(k2, z28, z24); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z25, z27, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1856))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z25, z28, z26); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1888))); + z25 = _mm512_slli_epi64(z28, 32); + z26 = _mm512_srli_epi64(z27, 32); + z25 = _mm512_mask_blend_epi32(k1, z27, z25); + z26 = _mm512_mask_blend_epi32(k2, z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z25 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z25 = _mm512_srai_epi16(z25, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z25 = _mm512_mullo_epi16(z25, z14); + z27 = _mm512_sub_epi16(z27, z23); + z28 = _mm512_sub_epi16(z28, z25); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 704))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 736))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 768))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 800))); + z4 = _mm512_unpacklo_epi32(z8, z5); + z5 = _mm512_unpackhi_epi32(z8, z5); + z6 = _mm512_unpacklo_epi32(z9, z7); + z7 = _mm512_unpackhi_epi32(z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1920))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1952))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1984))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2016))); + z23 = _mm512_unpacklo_epi32(z27, z24); + z24 = _mm512_unpackhi_epi32(z27, z24); + z25 = _mm512_unpacklo_epi32(z28, z26); + z26 = _mm512_unpackhi_epi32(z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 832))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 864))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 896))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 928))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z6 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z6 = _mm512_srai_epi16(z6, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z6 = _mm512_mullo_epi16(z6, z14); + z8 = _mm512_sub_epi16(z8, z4); + z9 = _mm512_sub_epi16(z9, z6); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2048))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2080))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2112))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2144))); + z23 = _mm512_unpacklo_epi64(z27, z24); + z24 = _mm512_unpackhi_epi64(z27, z24); + z25 = _mm512_unpacklo_epi64(z28, z26); + z26 = _mm512_unpackhi_epi64(z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z25 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z25 = _mm512_srai_epi16(z25, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z25 = _mm512_mullo_epi16(z25, z14); + z27 = _mm512_sub_epi16(z27, z23); + z28 = _mm512_sub_epi16(z28, z25); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 960))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z8, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 992))); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z9, z6, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1024))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z9, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1056))); + z8 = _mm512_sub_epi16(z4, z5); + z9 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_add_epi16(z4, z5); + z6 = _mm512_add_epi16(z6, z7); + z5 = _mm512_mullo_epi16(z8, z12); + z7 = _mm512_mullo_epi16(z9, z13); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z11); + z5 = _mm512_mulhi_epi16(z5, z14); + z7 = _mm512_mulhi_epi16(z7, z14); + z5 = _mm512_sub_epi16(z8, z5); + z7 = _mm512_sub_epi16(z9, z7); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z27, z23, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2176))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z27, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2208))); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z28, z25, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2240))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z28, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2272))); + z27 = _mm512_sub_epi16(z23, z24); + z28 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_add_epi16(z23, z24); + z25 = _mm512_add_epi16(z25, z26); + z24 = _mm512_mullo_epi16(z27, z12); + z26 = _mm512_mullo_epi16(z28, z13); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z11); + z24 = _mm512_mulhi_epi16(z24, z14); + z26 = _mm512_mulhi_epi16(z26, z14); + z24 = _mm512_sub_epi16(z27, z24); + z26 = _mm512_sub_epi16(z28, z26); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1088))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1120))); + z8 = _mm512_add_epi16(z4, z6); + z9 = _mm512_add_epi16(z5, z7); + z6 = _mm512_sub_epi16(z4, z6); + z7 = _mm512_sub_epi16(z5, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z5 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z5 = _mm512_srai_epi16(z5, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z5 = _mm512_mullo_epi16(z5, z14); + z4 = _mm512_sub_epi16(z8, z4); + z5 = _mm512_sub_epi16(z9, z5); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z6 = _mm512_sub_epi16(z6, z8); + z7 = _mm512_sub_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2304))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2336))); + z27 = _mm512_add_epi16(z23, z25); + z28 = _mm512_add_epi16(z24, z26); + z25 = _mm512_sub_epi16(z23, z25); + z26 = _mm512_sub_epi16(z24, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z24 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z24 = _mm512_srai_epi16(z24, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z24 = _mm512_mullo_epi16(z24, z14); + z23 = _mm512_sub_epi16(z27, z23); + z24 = _mm512_sub_epi16(z28, z24); + z27 = _mm512_mullo_epi16(z25, z12); + z28 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z27 = _mm512_mulhi_epi16(z27, z14); + z28 = _mm512_mulhi_epi16(z28, z14); + z25 = _mm512_sub_epi16(z25, z27); + z26 = _mm512_sub_epi16(z26, z28); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1184))); + z8 = _mm512_sub_epi16(z0, z4); + z9 = _mm512_sub_epi16(z1, z5); + z0 = _mm512_add_epi16(z0, z4); + z1 = _mm512_add_epi16(z1, z5); + z4 = _mm512_mullo_epi16(z8, z12); + z5 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z4 = _mm512_mulhi_epi16(z4, z14); + z5 = _mm512_mulhi_epi16(z5, z14); + z4 = _mm512_sub_epi16(z8, z4); + z5 = _mm512_sub_epi16(z9, z5); + z8 = _mm512_sub_epi16(z2, z6); + z9 = _mm512_sub_epi16(z3, z7); + z2 = _mm512_add_epi16(z2, z6); + z3 = _mm512_add_epi16(z3, z7); + z6 = _mm512_mullo_epi16(z8, z12); + z7 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z7 = _mm512_mulhi_epi16(z7, z14); + z6 = _mm512_sub_epi16(z8, z6); + z7 = _mm512_sub_epi16(z9, z7); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2400))); + z27 = _mm512_sub_epi16(z19, z23); + z28 = _mm512_sub_epi16(z20, z24); + z19 = _mm512_add_epi16(z19, z23); + z20 = _mm512_add_epi16(z20, z24); + z23 = _mm512_mullo_epi16(z27, z12); + z24 = _mm512_mullo_epi16(z28, z12); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z10); + z23 = _mm512_mulhi_epi16(z23, z14); + z24 = _mm512_mulhi_epi16(z24, z14); + z23 = _mm512_sub_epi16(z27, z23); + z24 = _mm512_sub_epi16(z28, z24); + z27 = _mm512_sub_epi16(z21, z25); + z28 = _mm512_sub_epi16(z22, z26); + z21 = _mm512_add_epi16(z21, z25); + z22 = _mm512_add_epi16(z22, z26); + z25 = _mm512_mullo_epi16(z27, z12); + z26 = _mm512_mullo_epi16(z28, z12); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z10); + z25 = _mm512_mulhi_epi16(z25, z14); + z26 = _mm512_mulhi_epi16(z26, z14); + z25 = _mm512_sub_epi16(z27, z25); + z26 = _mm512_sub_epi16(z28, z26); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2432))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2464))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2496))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2528))); + z8 = _mm512_sub_epi16(z0, z19); + z9 = _mm512_sub_epi16(z1, z20); + z0 = _mm512_add_epi16(z0, z19); + z1 = _mm512_add_epi16(z1, z20); + z19 = _mm512_mullo_epi16(z8, z12); + z20 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z19 = _mm512_mulhi_epi16(z19, z14); + z20 = _mm512_mulhi_epi16(z20, z14); + z19 = _mm512_sub_epi16(z8, z19); + z20 = _mm512_sub_epi16(z9, z20); + z8 = _mm512_add_epi16(z2, z21); + z9 = _mm512_add_epi16(z3, z22); + z21 = _mm512_sub_epi16(z2, z21); + z22 = _mm512_sub_epi16(z3, z22); + z2 = _mm512_mulhi_epi16(z8, z15); + z3 = _mm512_mulhi_epi16(z9, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z8, z2); + z3 = _mm512_sub_epi16(z9, z3); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z21 = _mm512_sub_epi16(z21, z8); + z22 = _mm512_sub_epi16(z22, z9); + z8 = _mm512_sub_epi16(z4, z23); + z9 = _mm512_sub_epi16(z5, z24); + z4 = _mm512_add_epi16(z4, z23); + z5 = _mm512_add_epi16(z5, z24); + z23 = _mm512_mullo_epi16(z8, z12); + z24 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z23 = _mm512_mulhi_epi16(z23, z14); + z24 = _mm512_mulhi_epi16(z24, z14); + z23 = _mm512_sub_epi16(z8, z23); + z24 = _mm512_sub_epi16(z9, z24); + z8 = _mm512_add_epi16(z6, z25); + z9 = _mm512_add_epi16(z7, z26); + z25 = _mm512_sub_epi16(z6, z25); + z26 = _mm512_sub_epi16(z7, z26); + z6 = _mm512_mulhi_epi16(z8, z15); + z7 = _mm512_mulhi_epi16(z9, z15); + z6 = _mm512_srai_epi16(z6, 10); + z7 = _mm512_srai_epi16(z7, 10); + z6 = _mm512_mullo_epi16(z6, z14); + z7 = _mm512_mullo_epi16(z7, z14); + z6 = _mm512_sub_epi16(z8, z6); + z7 = _mm512_sub_epi16(z9, z7); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z25 = _mm512_sub_epi16(z25, z8); + z26 = _mm512_sub_epi16(z26, z9); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z0 = _mm512_mulhi_epi16(z0, z11); + z1 = _mm512_mulhi_epi16(z1, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z0, z8); + z1 = _mm512_sub_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z2, z13); + z9 = _mm512_mullo_epi16(z3, z13); + z2 = _mm512_mulhi_epi16(z2, z11); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z2 = _mm512_sub_epi16(z2, z8); + z3 = _mm512_sub_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z4, z13); + z9 = _mm512_mullo_epi16(z5, z13); + z4 = _mm512_mulhi_epi16(z4, z11); + z5 = _mm512_mulhi_epi16(z5, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z4 = _mm512_sub_epi16(z4, z8); + z5 = _mm512_sub_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z6, z13); + z9 = _mm512_mullo_epi16(z7, z13); + z6 = _mm512_mulhi_epi16(z6, z11); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z6 = _mm512_sub_epi16(z6, z8); + z7 = _mm512_sub_epi16(z7, z9); + z8 = _mm512_mullo_epi16(z19, z13); + z9 = _mm512_mullo_epi16(z20, z13); + z19 = _mm512_mulhi_epi16(z19, z11); + z20 = _mm512_mulhi_epi16(z20, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z19 = _mm512_sub_epi16(z19, z8); + z20 = _mm512_sub_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z21, z13); + z9 = _mm512_mullo_epi16(z22, z13); + z21 = _mm512_mulhi_epi16(z21, z11); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z21 = _mm512_sub_epi16(z21, z8); + z22 = _mm512_sub_epi16(z22, z9); + z8 = _mm512_mullo_epi16(z23, z13); + z9 = _mm512_mullo_epi16(z24, z13); + z23 = _mm512_mulhi_epi16(z23, z11); + z24 = _mm512_mulhi_epi16(z24, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z23 = _mm512_sub_epi16(z23, z8); + z24 = _mm512_sub_epi16(z24, z9); + z8 = _mm512_mullo_epi16(z25, z13); + z9 = _mm512_mullo_epi16(z26, z13); + z25 = _mm512_mulhi_epi16(z25, z11); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z25 = _mm512_sub_epi16(z25, z8); + z26 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z7, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z19)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_extracti64x4_epi64( + z19, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z20)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_extracti64x4_epi64( + z20, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_castsi512_si256(z21)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64( + z21, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_castsi512_si256(z22)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64( + z22, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z23)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_extracti64x4_epi64( + z23, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z24)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_extracti64x4_epi64( + z24, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_castsi512_si256(z25)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64( + z25, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_castsi512_si256(z26)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64( + z26, 1)); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r10, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rax, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rax, 64)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rax, 128)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z4 = _mm512_add_epi16(z4, z0); + z5 = _mm512_add_epi16(z5, z1); + z6 = _mm512_add_epi16(z6, z2); + z7 = _mm512_add_epi16(z7, z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r10, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r10, 448)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rax, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rax, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rax, 448)); + z4 = _mm512_add_epi16(z4, z0); + z5 = _mm512_add_epi16(z5, z1); + z6 = _mm512_add_epi16(z6, z2); + z7 = _mm512_add_epi16(z7, z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rdx, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rdx, 320), z1); + _mm512_storeu_si512((void*)WC_PW(rdx, 384), z2); + _mm512_storeu_si512((void*)WC_PW(rdx, 448), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r9, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR(r9, 64)); + z6 = _mm512_loadu_si512((const void*)WC_PR(r9, 128)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r9, 192)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rsi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rsi, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z4 = _mm512_loadu_si512((const void*)WC_PR(r9, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(r9, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(r9, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(r9, 448)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rsi, 320), z1); + _mm512_storeu_si512((void*)WC_PW(rsi, 384), z2); + _mm512_storeu_si512((void*)WC_PW(rsi, 448), z3); + goto L_encap_avx512_calc_done; +L_encap_avx512_vp_single: + rbp = (word64)((word64)(size_t)L_mlkem_avx512_zetas_inv); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 0))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z0, z9, z1); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 32))); + z0 = _mm512_slli_epi64(z9, 32); + z1 = _mm512_srli_epi64(z8, 32); + z0 = _mm512_mask_blend_epi32(k1, z8, z0); + z1 = _mm512_mask_blend_epi32(k2, z9, z1); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z2, z8, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 64))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 96))); + z2 = _mm512_slli_epi64(z9, 32); + z3 = _mm512_srli_epi64(z8, 32); + z2 = _mm512_mask_blend_epi32(k1, z8, z2); + z3 = _mm512_mask_blend_epi32(k2, z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z2 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z2 = _mm512_srai_epi16(z2, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z2 = _mm512_mullo_epi16(z2, z14); + z8 = _mm512_sub_epi16(z8, z0); + z9 = _mm512_sub_epi16(z9, z2); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z19, z27, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1216))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z19, z28, z20); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1248))); + z19 = _mm512_slli_epi64(z28, 32); + z20 = _mm512_srli_epi64(z27, 32); + z19 = _mm512_mask_blend_epi32(k1, z27, z19); + z20 = _mm512_mask_blend_epi32(k2, z28, z20); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z21, z27, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1280))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z21, z28, z22); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1312))); + z21 = _mm512_slli_epi64(z28, 32); + z22 = _mm512_srli_epi64(z27, 32); + z21 = _mm512_mask_blend_epi32(k1, z27, z21); + z22 = _mm512_mask_blend_epi32(k2, z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z21 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z21 = _mm512_srai_epi16(z21, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z21 = _mm512_mullo_epi16(z21, z14); + z27 = _mm512_sub_epi16(z27, z19); + z28 = _mm512_sub_epi16(z28, z21); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 160))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 192))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 224))); + z0 = _mm512_unpacklo_epi32(z8, z1); + z1 = _mm512_unpackhi_epi32(z8, z1); + z2 = _mm512_unpacklo_epi32(z9, z3); + z3 = _mm512_unpackhi_epi32(z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1376))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1408))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1440))); + z19 = _mm512_unpacklo_epi32(z27, z20); + z20 = _mm512_unpackhi_epi32(z27, z20); + z21 = _mm512_unpacklo_epi32(z28, z22); + z22 = _mm512_unpackhi_epi32(z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 352))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z2 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z2 = _mm512_srai_epi16(z2, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z2 = _mm512_mullo_epi16(z2, z14); + z8 = _mm512_sub_epi16(z8, z0); + z9 = _mm512_sub_epi16(z9, z2); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1568))); + z19 = _mm512_unpacklo_epi64(z27, z20); + z20 = _mm512_unpackhi_epi64(z27, z20); + z21 = _mm512_unpacklo_epi64(z28, z22); + z22 = _mm512_unpackhi_epi64(z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z21 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z21 = _mm512_srai_epi16(z21, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z21 = _mm512_mullo_epi16(z21, z14); + z27 = _mm512_sub_epi16(z27, z19); + z28 = _mm512_sub_epi16(z28, z21); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 384))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z8, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 416))); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z9, z2, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 448))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z9, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 480))); + z8 = _mm512_sub_epi16(z0, z1); + z9 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_add_epi16(z0, z1); + z2 = _mm512_add_epi16(z2, z3); + z1 = _mm512_mullo_epi16(z8, z12); + z3 = _mm512_mullo_epi16(z9, z13); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z11); + z1 = _mm512_mulhi_epi16(z1, z14); + z3 = _mm512_mulhi_epi16(z3, z14); + z1 = _mm512_sub_epi16(z8, z1); + z3 = _mm512_sub_epi16(z9, z3); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z27, z19, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1600))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z27, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1632))); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z28, z21, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1664))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z28, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1696))); + z27 = _mm512_sub_epi16(z19, z20); + z28 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_add_epi16(z19, z20); + z21 = _mm512_add_epi16(z21, z22); + z20 = _mm512_mullo_epi16(z27, z12); + z22 = _mm512_mullo_epi16(z28, z13); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z11); + z20 = _mm512_mulhi_epi16(z20, z14); + z22 = _mm512_mulhi_epi16(z22, z14); + z20 = _mm512_sub_epi16(z27, z20); + z22 = _mm512_sub_epi16(z28, z22); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 512))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 544))); + z8 = _mm512_add_epi16(z0, z2); + z9 = _mm512_add_epi16(z1, z3); + z2 = _mm512_sub_epi16(z0, z2); + z3 = _mm512_sub_epi16(z1, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z1 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z8, z0); + z1 = _mm512_sub_epi16(z9, z1); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z2 = _mm512_sub_epi16(z2, z8); + z3 = _mm512_sub_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1728))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1760))); + z27 = _mm512_add_epi16(z19, z21); + z28 = _mm512_add_epi16(z20, z22); + z21 = _mm512_sub_epi16(z19, z21); + z22 = _mm512_sub_epi16(z20, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z20 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z20 = _mm512_srai_epi16(z20, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z20 = _mm512_mullo_epi16(z20, z14); + z19 = _mm512_sub_epi16(z27, z19); + z20 = _mm512_sub_epi16(z28, z20); + z27 = _mm512_mullo_epi16(z21, z12); + z28 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z27 = _mm512_mulhi_epi16(z27, z14); + z28 = _mm512_mulhi_epi16(z28, z14); + z21 = _mm512_sub_epi16(z21, z27); + z22 = _mm512_sub_epi16(z22, z28); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 576))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z4, z9, z5); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 608))); + z4 = _mm512_slli_epi64(z9, 32); + z5 = _mm512_srli_epi64(z8, 32); + z4 = _mm512_mask_blend_epi32(k1, z8, z4); + z5 = _mm512_mask_blend_epi32(k2, z9, z5); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z6, z8, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 640))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 672))); + z6 = _mm512_slli_epi64(z9, 32); + z7 = _mm512_srli_epi64(z8, 32); + z6 = _mm512_mask_blend_epi32(k1, z8, z6); + z7 = _mm512_mask_blend_epi32(k2, z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z6 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z6 = _mm512_srai_epi16(z6, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z6 = _mm512_mullo_epi16(z6, z14); + z8 = _mm512_sub_epi16(z8, z4); + z9 = _mm512_sub_epi16(z9, z6); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z23, z27, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1792))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z23, z28, z24); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1824))); + z23 = _mm512_slli_epi64(z28, 32); + z24 = _mm512_srli_epi64(z27, 32); + z23 = _mm512_mask_blend_epi32(k1, z27, z23); + z24 = _mm512_mask_blend_epi32(k2, z28, z24); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z25, z27, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1856))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z25, z28, z26); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1888))); + z25 = _mm512_slli_epi64(z28, 32); + z26 = _mm512_srli_epi64(z27, 32); + z25 = _mm512_mask_blend_epi32(k1, z27, z25); + z26 = _mm512_mask_blend_epi32(k2, z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z25 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z25 = _mm512_srai_epi16(z25, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z25 = _mm512_mullo_epi16(z25, z14); + z27 = _mm512_sub_epi16(z27, z23); + z28 = _mm512_sub_epi16(z28, z25); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 704))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 736))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 768))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 800))); + z4 = _mm512_unpacklo_epi32(z8, z5); + z5 = _mm512_unpackhi_epi32(z8, z5); + z6 = _mm512_unpacklo_epi32(z9, z7); + z7 = _mm512_unpackhi_epi32(z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1920))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1952))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1984))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2016))); + z23 = _mm512_unpacklo_epi32(z27, z24); + z24 = _mm512_unpackhi_epi32(z27, z24); + z25 = _mm512_unpacklo_epi32(z28, z26); + z26 = _mm512_unpackhi_epi32(z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 832))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 864))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 896))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 928))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z6 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z6 = _mm512_srai_epi16(z6, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z6 = _mm512_mullo_epi16(z6, z14); + z8 = _mm512_sub_epi16(z8, z4); + z9 = _mm512_sub_epi16(z9, z6); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2048))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2080))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2112))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2144))); + z23 = _mm512_unpacklo_epi64(z27, z24); + z24 = _mm512_unpackhi_epi64(z27, z24); + z25 = _mm512_unpacklo_epi64(z28, z26); + z26 = _mm512_unpackhi_epi64(z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z25 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z25 = _mm512_srai_epi16(z25, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z25 = _mm512_mullo_epi16(z25, z14); + z27 = _mm512_sub_epi16(z27, z23); + z28 = _mm512_sub_epi16(z28, z25); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 960))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z8, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 992))); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z9, z6, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1024))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z9, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1056))); + z8 = _mm512_sub_epi16(z4, z5); + z9 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_add_epi16(z4, z5); + z6 = _mm512_add_epi16(z6, z7); + z5 = _mm512_mullo_epi16(z8, z12); + z7 = _mm512_mullo_epi16(z9, z13); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z11); + z5 = _mm512_mulhi_epi16(z5, z14); + z7 = _mm512_mulhi_epi16(z7, z14); + z5 = _mm512_sub_epi16(z8, z5); + z7 = _mm512_sub_epi16(z9, z7); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z27, z23, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2176))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z27, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2208))); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z28, z25, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2240))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z28, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2272))); + z27 = _mm512_sub_epi16(z23, z24); + z28 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_add_epi16(z23, z24); + z25 = _mm512_add_epi16(z25, z26); + z24 = _mm512_mullo_epi16(z27, z12); + z26 = _mm512_mullo_epi16(z28, z13); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z11); + z24 = _mm512_mulhi_epi16(z24, z14); + z26 = _mm512_mulhi_epi16(z26, z14); + z24 = _mm512_sub_epi16(z27, z24); + z26 = _mm512_sub_epi16(z28, z26); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1088))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1120))); + z8 = _mm512_add_epi16(z4, z6); + z9 = _mm512_add_epi16(z5, z7); + z6 = _mm512_sub_epi16(z4, z6); + z7 = _mm512_sub_epi16(z5, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z5 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z5 = _mm512_srai_epi16(z5, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z5 = _mm512_mullo_epi16(z5, z14); + z4 = _mm512_sub_epi16(z8, z4); + z5 = _mm512_sub_epi16(z9, z5); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z6 = _mm512_sub_epi16(z6, z8); + z7 = _mm512_sub_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2304))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2336))); + z27 = _mm512_add_epi16(z23, z25); + z28 = _mm512_add_epi16(z24, z26); + z25 = _mm512_sub_epi16(z23, z25); + z26 = _mm512_sub_epi16(z24, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z24 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z24 = _mm512_srai_epi16(z24, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z24 = _mm512_mullo_epi16(z24, z14); + z23 = _mm512_sub_epi16(z27, z23); + z24 = _mm512_sub_epi16(z28, z24); + z27 = _mm512_mullo_epi16(z25, z12); + z28 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z27 = _mm512_mulhi_epi16(z27, z14); + z28 = _mm512_mulhi_epi16(z28, z14); + z25 = _mm512_sub_epi16(z25, z27); + z26 = _mm512_sub_epi16(z26, z28); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 1184))); + z8 = _mm512_sub_epi16(z0, z4); + z9 = _mm512_sub_epi16(z1, z5); + z0 = _mm512_add_epi16(z0, z4); + z1 = _mm512_add_epi16(z1, z5); + z4 = _mm512_mullo_epi16(z8, z12); + z5 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z4 = _mm512_mulhi_epi16(z4, z14); + z5 = _mm512_mulhi_epi16(z5, z14); + z4 = _mm512_sub_epi16(z8, z4); + z5 = _mm512_sub_epi16(z9, z5); + z8 = _mm512_sub_epi16(z2, z6); + z9 = _mm512_sub_epi16(z3, z7); + z2 = _mm512_add_epi16(z2, z6); + z3 = _mm512_add_epi16(z3, z7); + z6 = _mm512_mullo_epi16(z8, z12); + z7 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z7 = _mm512_mulhi_epi16(z7, z14); + z6 = _mm512_sub_epi16(z8, z6); + z7 = _mm512_sub_epi16(z9, z7); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2400))); + z27 = _mm512_sub_epi16(z19, z23); + z28 = _mm512_sub_epi16(z20, z24); + z19 = _mm512_add_epi16(z19, z23); + z20 = _mm512_add_epi16(z20, z24); + z23 = _mm512_mullo_epi16(z27, z12); + z24 = _mm512_mullo_epi16(z28, z12); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z10); + z23 = _mm512_mulhi_epi16(z23, z14); + z24 = _mm512_mulhi_epi16(z24, z14); + z23 = _mm512_sub_epi16(z27, z23); + z24 = _mm512_sub_epi16(z28, z24); + z27 = _mm512_sub_epi16(z21, z25); + z28 = _mm512_sub_epi16(z22, z26); + z21 = _mm512_add_epi16(z21, z25); + z22 = _mm512_add_epi16(z22, z26); + z25 = _mm512_mullo_epi16(z27, z12); + z26 = _mm512_mullo_epi16(z28, z12); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z10); + z25 = _mm512_mulhi_epi16(z25, z14); + z26 = _mm512_mulhi_epi16(z26, z14); + z25 = _mm512_sub_epi16(z27, z25); + z26 = _mm512_sub_epi16(z28, z26); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2432))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2464))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2496))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(rbp, + 2528))); + z8 = _mm512_sub_epi16(z0, z19); + z9 = _mm512_sub_epi16(z1, z20); + z0 = _mm512_add_epi16(z0, z19); + z1 = _mm512_add_epi16(z1, z20); + z19 = _mm512_mullo_epi16(z8, z12); + z20 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z19 = _mm512_mulhi_epi16(z19, z14); + z20 = _mm512_mulhi_epi16(z20, z14); + z19 = _mm512_sub_epi16(z8, z19); + z20 = _mm512_sub_epi16(z9, z20); + z8 = _mm512_add_epi16(z2, z21); + z9 = _mm512_add_epi16(z3, z22); + z21 = _mm512_sub_epi16(z2, z21); + z22 = _mm512_sub_epi16(z3, z22); + z2 = _mm512_mulhi_epi16(z8, z15); + z3 = _mm512_mulhi_epi16(z9, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z8, z2); + z3 = _mm512_sub_epi16(z9, z3); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z21 = _mm512_sub_epi16(z21, z8); + z22 = _mm512_sub_epi16(z22, z9); + z8 = _mm512_sub_epi16(z4, z23); + z9 = _mm512_sub_epi16(z5, z24); + z4 = _mm512_add_epi16(z4, z23); + z5 = _mm512_add_epi16(z5, z24); + z23 = _mm512_mullo_epi16(z8, z12); + z24 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z23 = _mm512_mulhi_epi16(z23, z14); + z24 = _mm512_mulhi_epi16(z24, z14); + z23 = _mm512_sub_epi16(z8, z23); + z24 = _mm512_sub_epi16(z9, z24); + z8 = _mm512_add_epi16(z6, z25); + z9 = _mm512_add_epi16(z7, z26); + z25 = _mm512_sub_epi16(z6, z25); + z26 = _mm512_sub_epi16(z7, z26); + z6 = _mm512_mulhi_epi16(z8, z15); + z7 = _mm512_mulhi_epi16(z9, z15); + z6 = _mm512_srai_epi16(z6, 10); + z7 = _mm512_srai_epi16(z7, 10); + z6 = _mm512_mullo_epi16(z6, z14); + z7 = _mm512_mullo_epi16(z7, z14); + z6 = _mm512_sub_epi16(z8, z6); + z7 = _mm512_sub_epi16(z9, z7); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z25 = _mm512_sub_epi16(z25, z8); + z26 = _mm512_sub_epi16(z26, z9); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z0 = _mm512_mulhi_epi16(z0, z11); + z1 = _mm512_mulhi_epi16(z1, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z0, z8); + z1 = _mm512_sub_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z2, z13); + z9 = _mm512_mullo_epi16(z3, z13); + z2 = _mm512_mulhi_epi16(z2, z11); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z2 = _mm512_sub_epi16(z2, z8); + z3 = _mm512_sub_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z4, z13); + z9 = _mm512_mullo_epi16(z5, z13); + z4 = _mm512_mulhi_epi16(z4, z11); + z5 = _mm512_mulhi_epi16(z5, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z4 = _mm512_sub_epi16(z4, z8); + z5 = _mm512_sub_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z6, z13); + z9 = _mm512_mullo_epi16(z7, z13); + z6 = _mm512_mulhi_epi16(z6, z11); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z6 = _mm512_sub_epi16(z6, z8); + z7 = _mm512_sub_epi16(z7, z9); + z8 = _mm512_mullo_epi16(z19, z13); + z9 = _mm512_mullo_epi16(z20, z13); + z19 = _mm512_mulhi_epi16(z19, z11); + z20 = _mm512_mulhi_epi16(z20, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z19 = _mm512_sub_epi16(z19, z8); + z20 = _mm512_sub_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z21, z13); + z9 = _mm512_mullo_epi16(z22, z13); + z21 = _mm512_mulhi_epi16(z21, z11); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z21 = _mm512_sub_epi16(z21, z8); + z22 = _mm512_sub_epi16(z22, z9); + z8 = _mm512_mullo_epi16(z23, z13); + z9 = _mm512_mullo_epi16(z24, z13); + z23 = _mm512_mulhi_epi16(z23, z11); + z24 = _mm512_mulhi_epi16(z24, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z23 = _mm512_sub_epi16(z23, z8); + z24 = _mm512_sub_epi16(z24, z9); + z8 = _mm512_mullo_epi16(z25, z13); + z9 = _mm512_mullo_epi16(z26, z13); + z25 = _mm512_mulhi_epi16(z25, z11); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z25 = _mm512_sub_epi16(z25, z8); + z26 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 160), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 192), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 224), _mm512_extracti64x4_epi64(z7, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_castsi512_si256(z19)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 256), _mm512_extracti64x4_epi64( + z19, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_castsi512_si256(z20)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 288), _mm512_extracti64x4_epi64( + z20, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_castsi512_si256(z21)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 320), _mm512_extracti64x4_epi64( + z21, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_castsi512_si256(z22)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 352), _mm512_extracti64x4_epi64( + z22, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_castsi512_si256(z23)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 384), _mm512_extracti64x4_epi64( + z23, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_castsi512_si256(z24)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 416), _mm512_extracti64x4_epi64( + z24, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_castsi512_si256(z25)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 448), _mm512_extracti64x4_epi64( + z25, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_castsi512_si256(z26)); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 480), _mm512_extracti64x4_epi64( + z26, 1)); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r10, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r10, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rax, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rax, 64)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rax, 128)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rax, 192)); + z4 = _mm512_add_epi16(z4, z0); + z5 = _mm512_add_epi16(z5, z1); + z6 = _mm512_add_epi16(z6, z2); + z7 = _mm512_add_epi16(z7, z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdx, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdx, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdx, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdx, 192)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rdx, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdx, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdx, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdx, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(r10, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(r10, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(r10, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(r10, 448)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rax, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rax, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rax, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rax, 448)); + z4 = _mm512_add_epi16(z4, z0); + z5 = _mm512_add_epi16(z5, z1); + z6 = _mm512_add_epi16(z6, z2); + z7 = _mm512_add_epi16(z7, z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdx, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdx, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdx, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdx, 448)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rdx, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rdx, 320), z1); + _mm512_storeu_si512((void*)WC_PW(rdx, 384), z2); + _mm512_storeu_si512((void*)WC_PW(rdx, 448), z3); +L_encap_avx512_calc_done: + ; +} + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw") +WOLFSSL_LOCAL void mlkem_decapsulate_avx512(const sword16* priv, sword16* mp, + sword16* bp, const sword16* vp, int kp) +{ + word64 rdi, rsi, rdx, rcx, r8, r12, rax, r9, r10 = 0, r11 = 0; + __m512i z0 = _mm512_setzero_si512(), z1 = _mm512_setzero_si512(), + z2 = _mm512_setzero_si512(), z3 = _mm512_setzero_si512(), + z4 = _mm512_setzero_si512(), z5 = _mm512_setzero_si512(), + z6 = _mm512_setzero_si512(), z7 = _mm512_setzero_si512(), + z8 = _mm512_setzero_si512(), z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(), + z12 = _mm512_setzero_si512(), z13 = _mm512_setzero_si512(), z14, + z15, z16 = _mm512_setzero_si512(), z17, z18, + z19 = _mm512_setzero_si512(), z20 = _mm512_setzero_si512(), + z21 = _mm512_setzero_si512(), z22 = _mm512_setzero_si512(), + z23 = _mm512_setzero_si512(), z24 = _mm512_setzero_si512(), + z25 = _mm512_setzero_si512(), z26 = _mm512_setzero_si512(), + z27 = _mm512_setzero_si512(), z28 = _mm512_setzero_si512(); + __mmask16 k1, k2; + __mmask32 k3, k4; + + rdi = (word64)(size_t)priv; + rsi = (word64)(size_t)mp; + rdx = (word64)(size_t)bp; + rcx = (word64)(size_t)vp; + r8 = (word64)(word32)kp; + + r12 = (word64)(0xd01); + z14 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r12)); + z14 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z14)); + r12 = (word64)(0x4ebf); + z15 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r12)); + z15 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z15)); + r12 = (word32)(0xaaaa); + k1 = (__mmask16)(word32)r12; + r12 = (word32)(0x5555); + k2 = (__mmask16)(word32)r12; + r12 = (word32)(0xaaaaaaaa); + k3 = (__mmask32)(word32)r12; + r12 = (word32)(0x55555555); + k4 = (__mmask32)(word32)r12; + z17 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_avx512_perm20, 0)); + z18 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_avx512_perm31, 0)); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + r9 = (word64)(rdx); +L_decap_avx512_bp: + if ((sword64)(rax) < (sword64)(2)) { + goto L_decap_avx512_bp_odd; + } + r10 = (word64)(r9); + r10 = (word64)(r10 + 0x200); + r11 = (word64)((word64)(size_t)L_mlkem_avx512_zetas); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(r10, + 480)), 1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32))); + z8 = _mm512_mullo_epi16(z19, z12); + z9 = _mm512_mullo_epi16(z20, z12); + z19 = _mm512_mulhi_epi16(z19, z10); + z20 = _mm512_mulhi_epi16(z20, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_sub_epi16(z0, z8); + z20 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z2, z8); + z22 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z4, z8); + z24 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z6, z8); + z26 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96))); + z8 = _mm512_mullo_epi16(z4, z12); + z9 = _mm512_mullo_epi16(z5, z12); + z4 = _mm512_mulhi_epi16(z4, z10); + z5 = _mm512_mulhi_epi16(z5, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_sub_epi16(z0, z8); + z5 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z2, z8); + z7 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1280))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1312))); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z19, z8); + z24 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z21, z8); + z26 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 160))); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1376))); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 192))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 224))); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1408))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1440))); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 352))); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1568))); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 384))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 416))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 448))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 480))); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1600))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1632))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1664))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1696))); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 512))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 544))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 576))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 608))); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z0 = _mm512_sub_epi16(z1, z0); + z2 = _mm512_sub_epi16(z3, z2); + z1 = _mm512_sub_epi16(z8, z0); + z3 = _mm512_sub_epi16(z9, z2); + z8 = _mm512_add_epi16(z8, z0); + z9 = _mm512_add_epi16(z9, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 640))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 672))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 704))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 736))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z19, z8, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1728))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z19, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1760))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z21, z9, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1792))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z21, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1824))); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z19 = _mm512_sub_epi16(z20, z19); + z21 = _mm512_sub_epi16(z22, z21); + z20 = _mm512_sub_epi16(z8, z19); + z22 = _mm512_sub_epi16(z9, z21); + z8 = _mm512_add_epi16(z8, z19); + z9 = _mm512_add_epi16(z9, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1856))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1888))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1920))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1952))); + z19 = _mm512_unpacklo_epi64(z8, z20); + z20 = _mm512_unpackhi_epi64(z8, z20); + z21 = _mm512_unpacklo_epi64(z9, z22); + z22 = _mm512_unpackhi_epi64(z9, z22); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 768))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 800))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 832))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 864))); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z4 = _mm512_sub_epi16(z5, z4); + z6 = _mm512_sub_epi16(z7, z6); + z5 = _mm512_sub_epi16(z8, z4); + z7 = _mm512_sub_epi16(z9, z6); + z8 = _mm512_add_epi16(z8, z4); + z9 = _mm512_add_epi16(z9, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 896))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 928))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 960))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 992))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z23, z8, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1984))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z23, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2016))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z25, z9, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2048))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z25, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2080))); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z23 = _mm512_sub_epi16(z24, z23); + z25 = _mm512_sub_epi16(z26, z25); + z24 = _mm512_sub_epi16(z8, z23); + z26 = _mm512_sub_epi16(z9, z25); + z8 = _mm512_add_epi16(z8, z23); + z9 = _mm512_add_epi16(z9, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2112))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2144))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2176))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2208))); + z23 = _mm512_unpacklo_epi64(z8, z24); + z24 = _mm512_unpackhi_epi64(z8, z24); + z25 = _mm512_unpacklo_epi64(z9, z26); + z26 = _mm512_unpackhi_epi64(z9, z26); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1024))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1056))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1088))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1120))); + z8 = _mm512_slli_epi64(z1, 32); + z9 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z8); + z1 = _mm512_mask_blend_epi32(k2, z1, z9); + z8 = _mm512_slli_epi64(z3, 32); + z9 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z8); + z3 = _mm512_mask_blend_epi32(k2, z3, z9); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2240))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2272))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2304))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2336))); + z8 = _mm512_slli_epi64(z20, 32); + z9 = _mm512_srli_epi64(z19, 32); + z19 = _mm512_mask_blend_epi32(k1, z19, z8); + z20 = _mm512_mask_blend_epi32(k2, z20, z9); + z8 = _mm512_slli_epi64(z22, 32); + z9 = _mm512_srli_epi64(z21, 32); + z21 = _mm512_mask_blend_epi32(k1, z21, z8); + z22 = _mm512_mask_blend_epi32(k2, z22, z9); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1184))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1216))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1248))); + z8 = _mm512_slli_epi64(z5, 32); + z9 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z8); + z5 = _mm512_mask_blend_epi32(k2, z5, z9); + z8 = _mm512_slli_epi64(z7, 32); + z9 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z8); + z7 = _mm512_mask_blend_epi32(k2, z7, z9); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2400))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2432))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2464))); + z8 = _mm512_slli_epi64(z24, 32); + z9 = _mm512_srli_epi64(z23, 32); + z23 = _mm512_mask_blend_epi32(k1, z23, z8); + z24 = _mm512_mask_blend_epi32(k2, z24, z9); + z8 = _mm512_slli_epi64(z26, 32); + z9 = _mm512_srli_epi64(z25, 32); + z25 = _mm512_mask_blend_epi32(k1, z25, z8); + z26 = _mm512_mask_blend_epi32(k2, z26, z9); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = _mm512_unpacklo_epi32(z0, z1); + z9 = _mm512_unpackhi_epi32(z0, z1); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z9); + z1 = z18; + z1 = _mm512_permutex2var_epi64(z8, z1, z9); + z8 = _mm512_unpacklo_epi32(z2, z3); + z9 = _mm512_unpackhi_epi32(z2, z3); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z8, z2, z9); + z3 = z18; + z3 = _mm512_permutex2var_epi64(z8, z3, z9); + z8 = _mm512_unpacklo_epi32(z19, z20); + z9 = _mm512_unpackhi_epi32(z19, z20); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z8, z19, z9); + z20 = z18; + z20 = _mm512_permutex2var_epi64(z8, z20, z9); + z8 = _mm512_unpacklo_epi32(z21, z22); + z9 = _mm512_unpackhi_epi32(z21, z22); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z8, z21, z9); + z22 = z18; + z22 = _mm512_permutex2var_epi64(z8, z22, z9); + z8 = _mm512_unpacklo_epi32(z4, z5); + z9 = _mm512_unpackhi_epi32(z4, z5); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z9); + z5 = z18; + z5 = _mm512_permutex2var_epi64(z8, z5, z9); + z8 = _mm512_unpacklo_epi32(z6, z7); + z9 = _mm512_unpackhi_epi32(z6, z7); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z8, z6, z9); + z7 = z18; + z7 = _mm512_permutex2var_epi64(z8, z7, z9); + z8 = _mm512_unpacklo_epi32(z23, z24); + z9 = _mm512_unpackhi_epi32(z23, z24); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z8, z23, z9); + z24 = z18; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z8 = _mm512_unpacklo_epi32(z25, z26); + z9 = _mm512_unpackhi_epi32(z25, z26); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z8, z25, z9); + z26 = z18; + z26 = _mm512_permutex2var_epi64(z8, z26, z9); + z8 = _mm512_mulhi_epi16(z0, z15); + z9 = _mm512_mulhi_epi16(z1, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z0, z8); + z9 = _mm512_sub_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 0), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 0), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 32), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 32), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z2, z15); + z9 = _mm512_mulhi_epi16(z3, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 64), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 64), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 96), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 96), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z19, z15); + z9 = _mm512_mulhi_epi16(z20, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 256), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 256), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 288), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 288), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z21, z15); + z9 = _mm512_mulhi_epi16(z22, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 320), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 320), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 352), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 352), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z4, z15); + z9 = _mm512_mulhi_epi16(z5, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 128), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 128), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 160), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 160), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z6, z15); + z9 = _mm512_mulhi_epi16(z7, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 192), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 192), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 224), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 224), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z23, z15); + z9 = _mm512_mulhi_epi16(z24, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 384), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 384), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 416), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 416), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z25, z15); + z9 = _mm512_mulhi_epi16(z26, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 448), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 448), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 480), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r10, 480), _mm512_extracti64x4_epi64(z9, + 1)); + r9 = (word64)(r9 + 0x400); + rax = (word64)(rax - 2); + goto L_decap_avx512_bp; +L_decap_avx512_bp_odd: + if ((rax) != (1)) { + goto L_decap_avx512_bp_done; + } + r11 = (word64)((word64)(size_t)L_mlkem_avx512_zetas); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r9, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(r9, + 480)), 1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32))); + z8 = _mm512_mullo_epi16(z19, z12); + z9 = _mm512_mullo_epi16(z20, z12); + z19 = _mm512_mulhi_epi16(z19, z10); + z20 = _mm512_mulhi_epi16(z20, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_sub_epi16(z0, z8); + z20 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z2, z8); + z22 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z4, z8); + z24 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z6, z8); + z26 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_add_epi16(z6, z8); + z7 = _mm512_add_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96))); + z8 = _mm512_mullo_epi16(z4, z12); + z9 = _mm512_mullo_epi16(z5, z12); + z4 = _mm512_mulhi_epi16(z4, z10); + z5 = _mm512_mulhi_epi16(z5, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_sub_epi16(z0, z8); + z5 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z2, z8); + z7 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_add_epi16(z2, z8); + z3 = _mm512_add_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1280))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1312))); + z8 = _mm512_mullo_epi16(z23, z12); + z9 = _mm512_mullo_epi16(z24, z12); + z23 = _mm512_mulhi_epi16(z23, z10); + z24 = _mm512_mulhi_epi16(z24, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_sub_epi16(z19, z8); + z24 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z21, z8); + z26 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_add_epi16(z21, z8); + z22 = _mm512_add_epi16(z22, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 160))); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + z2 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z1, z9); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1376))); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + z21 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z20, z9); + z19 = _mm512_add_epi16(z19, z8); + z20 = _mm512_add_epi16(z20, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 192))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 224))); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z5, z9); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1408))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1440))); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + z25 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z24, z9); + z23 = _mm512_add_epi16(z23, z8); + z24 = _mm512_add_epi16(z24, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 352))); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1568))); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 384))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 416))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 448))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 480))); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1600))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1632))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1664))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1696))); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 512))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z0, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 544))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 576))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z2, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 608))); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z0 = _mm512_sub_epi16(z1, z0); + z2 = _mm512_sub_epi16(z3, z2); + z1 = _mm512_sub_epi16(z8, z0); + z3 = _mm512_sub_epi16(z9, z2); + z8 = _mm512_add_epi16(z8, z0); + z9 = _mm512_add_epi16(z9, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 640))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 672))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 704))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 736))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z19, z8, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1728))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z19, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1760))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z21, z9, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1792))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z21, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1824))); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z19 = _mm512_sub_epi16(z20, z19); + z21 = _mm512_sub_epi16(z22, z21); + z20 = _mm512_sub_epi16(z8, z19); + z22 = _mm512_sub_epi16(z9, z21); + z8 = _mm512_add_epi16(z8, z19); + z9 = _mm512_add_epi16(z9, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1856))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1888))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1920))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1952))); + z19 = _mm512_unpacklo_epi64(z8, z20); + z20 = _mm512_unpackhi_epi64(z8, z20); + z21 = _mm512_unpacklo_epi64(z9, z22); + z22 = _mm512_unpackhi_epi64(z9, z22); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 768))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z4, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 800))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 832))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z6, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 864))); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z4 = _mm512_sub_epi16(z5, z4); + z6 = _mm512_sub_epi16(z7, z6); + z5 = _mm512_sub_epi16(z8, z4); + z7 = _mm512_sub_epi16(z9, z6); + z8 = _mm512_add_epi16(z8, z4); + z9 = _mm512_add_epi16(z9, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 896))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 928))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 960))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 992))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z23, z8, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1984))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z23, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2016))); + z9 = z17; + z9 = _mm512_permutex2var_epi64(z25, z9, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2048))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z25, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2080))); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z23 = _mm512_sub_epi16(z24, z23); + z25 = _mm512_sub_epi16(z26, z25); + z24 = _mm512_sub_epi16(z8, z23); + z26 = _mm512_sub_epi16(z9, z25); + z8 = _mm512_add_epi16(z8, z23); + z9 = _mm512_add_epi16(z9, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2112))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2144))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2176))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2208))); + z23 = _mm512_unpacklo_epi64(z8, z24); + z24 = _mm512_unpackhi_epi64(z8, z24); + z25 = _mm512_unpacklo_epi64(z9, z26); + z26 = _mm512_unpackhi_epi64(z9, z26); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1024))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1056))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1088))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1120))); + z8 = _mm512_slli_epi64(z1, 32); + z9 = _mm512_srli_epi64(z0, 32); + z0 = _mm512_mask_blend_epi32(k1, z0, z8); + z1 = _mm512_mask_blend_epi32(k2, z1, z9); + z8 = _mm512_slli_epi64(z3, 32); + z9 = _mm512_srli_epi64(z2, 32); + z2 = _mm512_mask_blend_epi32(k1, z2, z8); + z3 = _mm512_mask_blend_epi32(k2, z3, z9); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z1, z8); + z9 = _mm512_sub_epi16(z3, z9); + z1 = _mm512_sub_epi16(z0, z8); + z3 = _mm512_sub_epi16(z2, z9); + z0 = _mm512_add_epi16(z0, z8); + z2 = _mm512_add_epi16(z2, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2240))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2272))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2304))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2336))); + z8 = _mm512_slli_epi64(z20, 32); + z9 = _mm512_srli_epi64(z19, 32); + z19 = _mm512_mask_blend_epi32(k1, z19, z8); + z20 = _mm512_mask_blend_epi32(k2, z20, z9); + z8 = _mm512_slli_epi64(z22, 32); + z9 = _mm512_srli_epi64(z21, 32); + z21 = _mm512_mask_blend_epi32(k1, z21, z8); + z22 = _mm512_mask_blend_epi32(k2, z22, z9); + z8 = _mm512_mullo_epi16(z20, z12); + z9 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z20, z8); + z9 = _mm512_sub_epi16(z22, z9); + z20 = _mm512_sub_epi16(z19, z8); + z22 = _mm512_sub_epi16(z21, z9); + z19 = _mm512_add_epi16(z19, z8); + z21 = _mm512_add_epi16(z21, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1184))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1216))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1248))); + z8 = _mm512_slli_epi64(z5, 32); + z9 = _mm512_srli_epi64(z4, 32); + z4 = _mm512_mask_blend_epi32(k1, z4, z8); + z5 = _mm512_mask_blend_epi32(k2, z5, z9); + z8 = _mm512_slli_epi64(z7, 32); + z9 = _mm512_srli_epi64(z6, 32); + z6 = _mm512_mask_blend_epi32(k1, z6, z8); + z7 = _mm512_mask_blend_epi32(k2, z7, z9); + z8 = _mm512_mullo_epi16(z5, z12); + z9 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z5, z8); + z9 = _mm512_sub_epi16(z7, z9); + z5 = _mm512_sub_epi16(z4, z8); + z7 = _mm512_sub_epi16(z6, z9); + z4 = _mm512_add_epi16(z4, z8); + z6 = _mm512_add_epi16(z6, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2400))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2432))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2464))); + z8 = _mm512_slli_epi64(z24, 32); + z9 = _mm512_srli_epi64(z23, 32); + z23 = _mm512_mask_blend_epi32(k1, z23, z8); + z24 = _mm512_mask_blend_epi32(k2, z24, z9); + z8 = _mm512_slli_epi64(z26, 32); + z9 = _mm512_srli_epi64(z25, 32); + z25 = _mm512_mask_blend_epi32(k1, z25, z8); + z26 = _mm512_mask_blend_epi32(k2, z26, z9); + z8 = _mm512_mullo_epi16(z24, z12); + z9 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z8 = _mm512_sub_epi16(z24, z8); + z9 = _mm512_sub_epi16(z26, z9); + z24 = _mm512_sub_epi16(z23, z8); + z26 = _mm512_sub_epi16(z25, z9); + z23 = _mm512_add_epi16(z23, z8); + z25 = _mm512_add_epi16(z25, z9); + z8 = _mm512_unpacklo_epi32(z0, z1); + z9 = _mm512_unpackhi_epi32(z0, z1); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z9); + z1 = z18; + z1 = _mm512_permutex2var_epi64(z8, z1, z9); + z8 = _mm512_unpacklo_epi32(z2, z3); + z9 = _mm512_unpackhi_epi32(z2, z3); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z8, z2, z9); + z3 = z18; + z3 = _mm512_permutex2var_epi64(z8, z3, z9); + z8 = _mm512_unpacklo_epi32(z19, z20); + z9 = _mm512_unpackhi_epi32(z19, z20); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z8, z19, z9); + z20 = z18; + z20 = _mm512_permutex2var_epi64(z8, z20, z9); + z8 = _mm512_unpacklo_epi32(z21, z22); + z9 = _mm512_unpackhi_epi32(z21, z22); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z8, z21, z9); + z22 = z18; + z22 = _mm512_permutex2var_epi64(z8, z22, z9); + z8 = _mm512_unpacklo_epi32(z4, z5); + z9 = _mm512_unpackhi_epi32(z4, z5); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z9); + z5 = z18; + z5 = _mm512_permutex2var_epi64(z8, z5, z9); + z8 = _mm512_unpacklo_epi32(z6, z7); + z9 = _mm512_unpackhi_epi32(z6, z7); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z8, z6, z9); + z7 = z18; + z7 = _mm512_permutex2var_epi64(z8, z7, z9); + z8 = _mm512_unpacklo_epi32(z23, z24); + z9 = _mm512_unpackhi_epi32(z23, z24); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z8, z23, z9); + z24 = z18; + z24 = _mm512_permutex2var_epi64(z8, z24, z9); + z8 = _mm512_unpacklo_epi32(z25, z26); + z9 = _mm512_unpackhi_epi32(z25, z26); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z8, z25, z9); + z26 = z18; + z26 = _mm512_permutex2var_epi64(z8, z26, z9); + z8 = _mm512_mulhi_epi16(z0, z15); + z9 = _mm512_mulhi_epi16(z1, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z0, z8); + z9 = _mm512_sub_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 0), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 0), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 32), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 32), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z2, z15); + z9 = _mm512_mulhi_epi16(z3, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z2, z8); + z9 = _mm512_sub_epi16(z3, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 64), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 64), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 96), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 96), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z19, z15); + z9 = _mm512_mulhi_epi16(z20, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z19, z8); + z9 = _mm512_sub_epi16(z20, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 256), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 256), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 288), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 288), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z21, z15); + z9 = _mm512_mulhi_epi16(z22, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z21, z8); + z9 = _mm512_sub_epi16(z22, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 320), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 320), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 352), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 352), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z4, z15); + z9 = _mm512_mulhi_epi16(z5, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z4, z8); + z9 = _mm512_sub_epi16(z5, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 128), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 128), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 160), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 160), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z6, z15); + z9 = _mm512_mulhi_epi16(z7, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z6, z8); + z9 = _mm512_sub_epi16(z7, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 192), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 192), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 224), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 224), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z23, z15); + z9 = _mm512_mulhi_epi16(z24, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z23, z8); + z9 = _mm512_sub_epi16(z24, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 384), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 384), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 416), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 416), _mm512_extracti64x4_epi64(z9, + 1)); + z8 = _mm512_mulhi_epi16(z25, z15); + z9 = _mm512_mulhi_epi16(z26, z15); + z8 = _mm512_srai_epi16(z8, 10); + z9 = _mm512_srai_epi16(z9, 10); + z8 = _mm512_mullo_epi16(z8, z14); + z9 = _mm512_mullo_epi16(z9, z14); + z8 = _mm512_sub_epi16(z25, z8); + z9 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 448), _mm512_castsi512_si256(z8)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 448), _mm512_extracti64x4_epi64(z8, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 480), _mm512_castsi512_si256(z9)); + _mm256_storeu_si256((__m256i*)WC_PW(r9, 480), _mm512_extracti64x4_epi64(z9, + 1)); +L_decap_avx512_bp_done: + r12 = (word64)(0xf301); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r12)); + z12 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z12)); + rax = (word64)((word64)(sword64)(sword32)(word32)r8); + r11 = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rdi = (word64)(rdi + 0x200); + rdx = (word64)(rdx + 0x200); + rax = (word64)(rax - 2); + if ((rax) == (0)) { + goto L_pointwise_acc_mont_end_decap_avx512; + } +L_pointwise_acc_mont_start_decap_avx512: + r11 = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rdi = (word64)(rdi + 0x200); + rdx = (word64)(rdx + 0x200); + rax = (word64)(rax - 1); + if ((sword64)(rax) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_decap_avx512; + } +L_pointwise_acc_mont_end_decap_avx512: + r11 = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 128))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 192)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 160))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 256))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 320)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 288))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z6 = _mm512_slli_epi32(z3, 16); + z7 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z6); + z3 = _mm512_mask_blend_epi16(k4, z3, z7); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z6 = _mm512_slli_epi32(z5, 16); + z7 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z6); + z5 = _mm512_mask_blend_epi16(k4, z5, z7); + z10 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 384))); + z10 = _mm512_inserti64x4(z10, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 448)), 1); + z11 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 416))); + z11 = _mm512_inserti64x4(z11, _mm256_loadu_si256((const __m256i*)WC_PR(r11, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z6 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z7 = _mm512_mulhi_epi16(z2, z4); + z8 = _mm512_mullo_epi16(z0, z12); + z9 = _mm512_mullo_epi16(z1, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z6, z8); + z1 = _mm512_sub_epi16(z7, z9); + z6 = _mm512_mullo_epi16(z0, z11); + z7 = _mm512_mulhi_epi16(z0, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z0 = _mm512_sub_epi16(z7, z6); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z6 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z7 = _mm512_mulhi_epi16(z3, z4); + z8 = _mm512_mullo_epi16(z1, z12); + z9 = _mm512_mullo_epi16(z2, z12); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z1 = _mm512_sub_epi16(z6, z8); + z2 = _mm512_sub_epi16(z7, z9); + z1 = _mm512_add_epi16(z1, z2); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z0 = _mm512_add_epi16(z0, z6); + z1 = _mm512_add_epi16(z1, z7); + z6 = _mm512_slli_epi32(z1, 16); + z7 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z6); + z1 = _mm512_mask_blend_epi16(k4, z1, z7); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rdi = (word64)(rdi + 0x200); + r11 = (word64)((word64)(size_t)L_mlkem_avx512_zetas_inv); + z0 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z0 = _mm512_inserti64x4(z0, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0)), 1); + z1 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z1 = _mm512_inserti64x4(z1, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32)), 1); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160)), 1); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z19 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z19 = _mm512_inserti64x4(z19, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256)), 1); + z20 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z20 = _mm512_inserti64x4(z20, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288)), 1); + z21 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320))); + z21 = _mm512_inserti64x4(z21, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z22 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352))); + z22 = _mm512_inserti64x4(z22, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z23 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z23 = _mm512_inserti64x4(z23, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384)), 1); + z24 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z24 = _mm512_inserti64x4(z24, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416)), 1); + z25 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448))); + z25 = _mm512_inserti64x4(z25, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z26 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480))); + z26 = _mm512_inserti64x4(z26, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z0, z8, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 0))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z0, z9, z1); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 32))); + z0 = _mm512_slli_epi64(z9, 32); + z1 = _mm512_srli_epi64(z8, 32); + z0 = _mm512_mask_blend_epi32(k1, z8, z0); + z1 = _mm512_mask_blend_epi32(k2, z9, z1); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z2, z8, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 64))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z2, z9, z3); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 96))); + z2 = _mm512_slli_epi64(z9, 32); + z3 = _mm512_srli_epi64(z8, 32); + z2 = _mm512_mask_blend_epi32(k1, z8, z2); + z3 = _mm512_mask_blend_epi32(k2, z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z2 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z2 = _mm512_srai_epi16(z2, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z2 = _mm512_mullo_epi16(z2, z14); + z8 = _mm512_sub_epi16(z8, z0); + z9 = _mm512_sub_epi16(z9, z2); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z19, z27, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1216))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z19, z28, z20); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1248))); + z19 = _mm512_slli_epi64(z28, 32); + z20 = _mm512_srli_epi64(z27, 32); + z19 = _mm512_mask_blend_epi32(k1, z27, z19); + z20 = _mm512_mask_blend_epi32(k2, z28, z20); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z21, z27, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1280))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z21, z28, z22); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1312))); + z21 = _mm512_slli_epi64(z28, 32); + z22 = _mm512_srli_epi64(z27, 32); + z21 = _mm512_mask_blend_epi32(k1, z27, z21); + z22 = _mm512_mask_blend_epi32(k2, z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z21 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z21 = _mm512_srai_epi16(z21, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z21 = _mm512_mullo_epi16(z21, z14); + z27 = _mm512_sub_epi16(z27, z19); + z28 = _mm512_sub_epi16(z28, z21); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 128))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 160))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 192))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 224))); + z0 = _mm512_unpacklo_epi32(z8, z1); + z1 = _mm512_unpackhi_epi32(z8, z1); + z2 = _mm512_unpacklo_epi32(z9, z3); + z3 = _mm512_unpackhi_epi32(z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1344))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1376))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1408))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1440))); + z19 = _mm512_unpacklo_epi32(z27, z20); + z20 = _mm512_unpackhi_epi32(z27, z20); + z21 = _mm512_unpacklo_epi32(z28, z22); + z22 = _mm512_unpackhi_epi32(z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 256))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 288))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 320))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 352))); + z0 = _mm512_unpacklo_epi64(z8, z1); + z1 = _mm512_unpackhi_epi64(z8, z1); + z2 = _mm512_unpacklo_epi64(z9, z3); + z3 = _mm512_unpackhi_epi64(z9, z3); + z8 = _mm512_add_epi16(z0, z1); + z9 = _mm512_add_epi16(z2, z3); + z1 = _mm512_sub_epi16(z0, z1); + z3 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z2 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z2 = _mm512_srai_epi16(z2, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z2 = _mm512_mullo_epi16(z2, z14); + z8 = _mm512_sub_epi16(z8, z0); + z9 = _mm512_sub_epi16(z9, z2); + z0 = _mm512_mullo_epi16(z1, z12); + z2 = _mm512_mullo_epi16(z3, z13); + z1 = _mm512_mulhi_epi16(z1, z10); + z3 = _mm512_mulhi_epi16(z3, z11); + z0 = _mm512_mulhi_epi16(z0, z14); + z2 = _mm512_mulhi_epi16(z2, z14); + z1 = _mm512_sub_epi16(z1, z0); + z3 = _mm512_sub_epi16(z3, z2); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1472))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1504))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1536))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1568))); + z19 = _mm512_unpacklo_epi64(z27, z20); + z20 = _mm512_unpackhi_epi64(z27, z20); + z21 = _mm512_unpacklo_epi64(z28, z22); + z22 = _mm512_unpackhi_epi64(z28, z22); + z27 = _mm512_add_epi16(z19, z20); + z28 = _mm512_add_epi16(z21, z22); + z20 = _mm512_sub_epi16(z19, z20); + z22 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z21 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z21 = _mm512_srai_epi16(z21, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z21 = _mm512_mullo_epi16(z21, z14); + z27 = _mm512_sub_epi16(z27, z19); + z28 = _mm512_sub_epi16(z28, z21); + z19 = _mm512_mullo_epi16(z20, z12); + z21 = _mm512_mullo_epi16(z22, z13); + z20 = _mm512_mulhi_epi16(z20, z10); + z22 = _mm512_mulhi_epi16(z22, z11); + z19 = _mm512_mulhi_epi16(z19, z14); + z21 = _mm512_mulhi_epi16(z21, z14); + z20 = _mm512_sub_epi16(z20, z19); + z22 = _mm512_sub_epi16(z22, z21); + z0 = z17; + z0 = _mm512_permutex2var_epi64(z8, z0, z1); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 384))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z8, z16, z1); + z1 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 416))); + z2 = z17; + z2 = _mm512_permutex2var_epi64(z9, z2, z3); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 448))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z9, z16, z3); + z3 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 480))); + z8 = _mm512_sub_epi16(z0, z1); + z9 = _mm512_sub_epi16(z2, z3); + z0 = _mm512_add_epi16(z0, z1); + z2 = _mm512_add_epi16(z2, z3); + z1 = _mm512_mullo_epi16(z8, z12); + z3 = _mm512_mullo_epi16(z9, z13); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z11); + z1 = _mm512_mulhi_epi16(z1, z14); + z3 = _mm512_mulhi_epi16(z3, z14); + z1 = _mm512_sub_epi16(z8, z1); + z3 = _mm512_sub_epi16(z9, z3); + z19 = z17; + z19 = _mm512_permutex2var_epi64(z27, z19, z20); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1600))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z27, z16, z20); + z20 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1632))); + z21 = z17; + z21 = _mm512_permutex2var_epi64(z28, z21, z22); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1664))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z28, z16, z22); + z22 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1696))); + z27 = _mm512_sub_epi16(z19, z20); + z28 = _mm512_sub_epi16(z21, z22); + z19 = _mm512_add_epi16(z19, z20); + z21 = _mm512_add_epi16(z21, z22); + z20 = _mm512_mullo_epi16(z27, z12); + z22 = _mm512_mullo_epi16(z28, z13); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z11); + z20 = _mm512_mulhi_epi16(z20, z14); + z22 = _mm512_mulhi_epi16(z22, z14); + z20 = _mm512_sub_epi16(z27, z20); + z22 = _mm512_sub_epi16(z28, z22); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 512))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 544))); + z8 = _mm512_add_epi16(z0, z2); + z9 = _mm512_add_epi16(z1, z3); + z2 = _mm512_sub_epi16(z0, z2); + z3 = _mm512_sub_epi16(z1, z3); + z0 = _mm512_mulhi_epi16(z8, z15); + z1 = _mm512_mulhi_epi16(z9, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z8, z0); + z1 = _mm512_sub_epi16(z9, z1); + z8 = _mm512_mullo_epi16(z2, z12); + z9 = _mm512_mullo_epi16(z3, z12); + z2 = _mm512_mulhi_epi16(z2, z10); + z3 = _mm512_mulhi_epi16(z3, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z2 = _mm512_sub_epi16(z2, z8); + z3 = _mm512_sub_epi16(z3, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1728))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1760))); + z27 = _mm512_add_epi16(z19, z21); + z28 = _mm512_add_epi16(z20, z22); + z21 = _mm512_sub_epi16(z19, z21); + z22 = _mm512_sub_epi16(z20, z22); + z19 = _mm512_mulhi_epi16(z27, z15); + z20 = _mm512_mulhi_epi16(z28, z15); + z19 = _mm512_srai_epi16(z19, 10); + z20 = _mm512_srai_epi16(z20, 10); + z19 = _mm512_mullo_epi16(z19, z14); + z20 = _mm512_mullo_epi16(z20, z14); + z19 = _mm512_sub_epi16(z27, z19); + z20 = _mm512_sub_epi16(z28, z20); + z27 = _mm512_mullo_epi16(z21, z12); + z28 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z27 = _mm512_mulhi_epi16(z27, z14); + z28 = _mm512_mulhi_epi16(z28, z14); + z21 = _mm512_sub_epi16(z21, z27); + z22 = _mm512_sub_epi16(z22, z28); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z4, z8, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 576))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z4, z9, z5); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 608))); + z4 = _mm512_slli_epi64(z9, 32); + z5 = _mm512_srli_epi64(z8, 32); + z4 = _mm512_mask_blend_epi32(k1, z8, z4); + z5 = _mm512_mask_blend_epi32(k2, z9, z5); + z8 = z17; + z8 = _mm512_permutex2var_epi64(z6, z8, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 640))); + z9 = z18; + z9 = _mm512_permutex2var_epi64(z6, z9, z7); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 672))); + z6 = _mm512_slli_epi64(z9, 32); + z7 = _mm512_srli_epi64(z8, 32); + z6 = _mm512_mask_blend_epi32(k1, z8, z6); + z7 = _mm512_mask_blend_epi32(k2, z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z6 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z6 = _mm512_srai_epi16(z6, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z6 = _mm512_mullo_epi16(z6, z14); + z8 = _mm512_sub_epi16(z8, z4); + z9 = _mm512_sub_epi16(z9, z6); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z23, z27, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1792))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z23, z28, z24); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1824))); + z23 = _mm512_slli_epi64(z28, 32); + z24 = _mm512_srli_epi64(z27, 32); + z23 = _mm512_mask_blend_epi32(k1, z27, z23); + z24 = _mm512_mask_blend_epi32(k2, z28, z24); + z27 = z17; + z27 = _mm512_permutex2var_epi64(z25, z27, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1856))); + z28 = z18; + z28 = _mm512_permutex2var_epi64(z25, z28, z26); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1888))); + z25 = _mm512_slli_epi64(z28, 32); + z26 = _mm512_srli_epi64(z27, 32); + z25 = _mm512_mask_blend_epi32(k1, z27, z25); + z26 = _mm512_mask_blend_epi32(k2, z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z25 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z25 = _mm512_srai_epi16(z25, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z25 = _mm512_mullo_epi16(z25, z14); + z27 = _mm512_sub_epi16(z27, z23); + z28 = _mm512_sub_epi16(z28, z25); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 704))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 736))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 768))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 800))); + z4 = _mm512_unpacklo_epi32(z8, z5); + z5 = _mm512_unpackhi_epi32(z8, z5); + z6 = _mm512_unpacklo_epi32(z9, z7); + z7 = _mm512_unpackhi_epi32(z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1920))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1952))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1984))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2016))); + z23 = _mm512_unpacklo_epi32(z27, z24); + z24 = _mm512_unpackhi_epi32(z27, z24); + z25 = _mm512_unpacklo_epi32(z28, z26); + z26 = _mm512_unpackhi_epi32(z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 832))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 864))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 896))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 928))); + z4 = _mm512_unpacklo_epi64(z8, z5); + z5 = _mm512_unpackhi_epi64(z8, z5); + z6 = _mm512_unpacklo_epi64(z9, z7); + z7 = _mm512_unpackhi_epi64(z9, z7); + z8 = _mm512_add_epi16(z4, z5); + z9 = _mm512_add_epi16(z6, z7); + z5 = _mm512_sub_epi16(z4, z5); + z7 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z6 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z6 = _mm512_srai_epi16(z6, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z6 = _mm512_mullo_epi16(z6, z14); + z8 = _mm512_sub_epi16(z8, z4); + z9 = _mm512_sub_epi16(z9, z6); + z4 = _mm512_mullo_epi16(z5, z12); + z6 = _mm512_mullo_epi16(z7, z13); + z5 = _mm512_mulhi_epi16(z5, z10); + z7 = _mm512_mulhi_epi16(z7, z11); + z4 = _mm512_mulhi_epi16(z4, z14); + z6 = _mm512_mulhi_epi16(z6, z14); + z5 = _mm512_sub_epi16(z5, z4); + z7 = _mm512_sub_epi16(z7, z6); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2048))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2080))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2112))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2144))); + z23 = _mm512_unpacklo_epi64(z27, z24); + z24 = _mm512_unpackhi_epi64(z27, z24); + z25 = _mm512_unpacklo_epi64(z28, z26); + z26 = _mm512_unpackhi_epi64(z28, z26); + z27 = _mm512_add_epi16(z23, z24); + z28 = _mm512_add_epi16(z25, z26); + z24 = _mm512_sub_epi16(z23, z24); + z26 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z25 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z25 = _mm512_srai_epi16(z25, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z25 = _mm512_mullo_epi16(z25, z14); + z27 = _mm512_sub_epi16(z27, z23); + z28 = _mm512_sub_epi16(z28, z25); + z23 = _mm512_mullo_epi16(z24, z12); + z25 = _mm512_mullo_epi16(z26, z13); + z24 = _mm512_mulhi_epi16(z24, z10); + z26 = _mm512_mulhi_epi16(z26, z11); + z23 = _mm512_mulhi_epi16(z23, z14); + z25 = _mm512_mulhi_epi16(z25, z14); + z24 = _mm512_sub_epi16(z24, z23); + z26 = _mm512_sub_epi16(z26, z25); + z4 = z17; + z4 = _mm512_permutex2var_epi64(z8, z4, z5); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 960))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z8, z16, z5); + z5 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 992))); + z6 = z17; + z6 = _mm512_permutex2var_epi64(z9, z6, z7); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1024))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z9, z16, z7); + z7 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1056))); + z8 = _mm512_sub_epi16(z4, z5); + z9 = _mm512_sub_epi16(z6, z7); + z4 = _mm512_add_epi16(z4, z5); + z6 = _mm512_add_epi16(z6, z7); + z5 = _mm512_mullo_epi16(z8, z12); + z7 = _mm512_mullo_epi16(z9, z13); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z11); + z5 = _mm512_mulhi_epi16(z5, z14); + z7 = _mm512_mulhi_epi16(z7, z14); + z5 = _mm512_sub_epi16(z8, z5); + z7 = _mm512_sub_epi16(z9, z7); + z23 = z17; + z23 = _mm512_permutex2var_epi64(z27, z23, z24); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2176))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z27, z16, z24); + z24 = z16; + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2208))); + z25 = z17; + z25 = _mm512_permutex2var_epi64(z28, z25, z26); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2240))); + z16 = z18; + z16 = _mm512_permutex2var_epi64(z28, z16, z26); + z26 = z16; + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2272))); + z27 = _mm512_sub_epi16(z23, z24); + z28 = _mm512_sub_epi16(z25, z26); + z23 = _mm512_add_epi16(z23, z24); + z25 = _mm512_add_epi16(z25, z26); + z24 = _mm512_mullo_epi16(z27, z12); + z26 = _mm512_mullo_epi16(z28, z13); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z11); + z24 = _mm512_mulhi_epi16(z24, z14); + z26 = _mm512_mulhi_epi16(z26, z14); + z24 = _mm512_sub_epi16(z27, z24); + z26 = _mm512_sub_epi16(z28, z26); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1088))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1120))); + z8 = _mm512_add_epi16(z4, z6); + z9 = _mm512_add_epi16(z5, z7); + z6 = _mm512_sub_epi16(z4, z6); + z7 = _mm512_sub_epi16(z5, z7); + z4 = _mm512_mulhi_epi16(z8, z15); + z5 = _mm512_mulhi_epi16(z9, z15); + z4 = _mm512_srai_epi16(z4, 10); + z5 = _mm512_srai_epi16(z5, 10); + z4 = _mm512_mullo_epi16(z4, z14); + z5 = _mm512_mullo_epi16(z5, z14); + z4 = _mm512_sub_epi16(z8, z4); + z5 = _mm512_sub_epi16(z9, z5); + z8 = _mm512_mullo_epi16(z6, z12); + z9 = _mm512_mullo_epi16(z7, z12); + z6 = _mm512_mulhi_epi16(z6, z10); + z7 = _mm512_mulhi_epi16(z7, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z6 = _mm512_sub_epi16(z6, z8); + z7 = _mm512_sub_epi16(z7, z9); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2304))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2336))); + z27 = _mm512_add_epi16(z23, z25); + z28 = _mm512_add_epi16(z24, z26); + z25 = _mm512_sub_epi16(z23, z25); + z26 = _mm512_sub_epi16(z24, z26); + z23 = _mm512_mulhi_epi16(z27, z15); + z24 = _mm512_mulhi_epi16(z28, z15); + z23 = _mm512_srai_epi16(z23, 10); + z24 = _mm512_srai_epi16(z24, 10); + z23 = _mm512_mullo_epi16(z23, z14); + z24 = _mm512_mullo_epi16(z24, z14); + z23 = _mm512_sub_epi16(z27, z23); + z24 = _mm512_sub_epi16(z28, z24); + z27 = _mm512_mullo_epi16(z25, z12); + z28 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z27 = _mm512_mulhi_epi16(z27, z14); + z28 = _mm512_mulhi_epi16(z28, z14); + z25 = _mm512_sub_epi16(z25, z27); + z26 = _mm512_sub_epi16(z26, z28); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1152))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 1184))); + z8 = _mm512_sub_epi16(z0, z4); + z9 = _mm512_sub_epi16(z1, z5); + z0 = _mm512_add_epi16(z0, z4); + z1 = _mm512_add_epi16(z1, z5); + z4 = _mm512_mullo_epi16(z8, z12); + z5 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z4 = _mm512_mulhi_epi16(z4, z14); + z5 = _mm512_mulhi_epi16(z5, z14); + z4 = _mm512_sub_epi16(z8, z4); + z5 = _mm512_sub_epi16(z9, z5); + z8 = _mm512_sub_epi16(z2, z6); + z9 = _mm512_sub_epi16(z3, z7); + z2 = _mm512_add_epi16(z2, z6); + z3 = _mm512_add_epi16(z3, z7); + z6 = _mm512_mullo_epi16(z8, z12); + z7 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z6 = _mm512_mulhi_epi16(z6, z14); + z7 = _mm512_mulhi_epi16(z7, z14); + z6 = _mm512_sub_epi16(z8, z6); + z7 = _mm512_sub_epi16(z9, z7); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2368))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2400))); + z27 = _mm512_sub_epi16(z19, z23); + z28 = _mm512_sub_epi16(z20, z24); + z19 = _mm512_add_epi16(z19, z23); + z20 = _mm512_add_epi16(z20, z24); + z23 = _mm512_mullo_epi16(z27, z12); + z24 = _mm512_mullo_epi16(z28, z12); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z10); + z23 = _mm512_mulhi_epi16(z23, z14); + z24 = _mm512_mulhi_epi16(z24, z14); + z23 = _mm512_sub_epi16(z27, z23); + z24 = _mm512_sub_epi16(z28, z24); + z27 = _mm512_sub_epi16(z21, z25); + z28 = _mm512_sub_epi16(z22, z26); + z21 = _mm512_add_epi16(z21, z25); + z22 = _mm512_add_epi16(z22, z26); + z25 = _mm512_mullo_epi16(z27, z12); + z26 = _mm512_mullo_epi16(z28, z12); + z27 = _mm512_mulhi_epi16(z27, z10); + z28 = _mm512_mulhi_epi16(z28, z10); + z25 = _mm512_mulhi_epi16(z25, z14); + z26 = _mm512_mulhi_epi16(z26, z14); + z25 = _mm512_sub_epi16(z27, z25); + z26 = _mm512_sub_epi16(z28, z26); + z10 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2432))); + z12 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2464))); + z11 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2496))); + z13 = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)WC_PR(r11, + 2528))); + z8 = _mm512_sub_epi16(z0, z19); + z9 = _mm512_sub_epi16(z1, z20); + z0 = _mm512_add_epi16(z0, z19); + z1 = _mm512_add_epi16(z1, z20); + z19 = _mm512_mullo_epi16(z8, z12); + z20 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z19 = _mm512_mulhi_epi16(z19, z14); + z20 = _mm512_mulhi_epi16(z20, z14); + z19 = _mm512_sub_epi16(z8, z19); + z20 = _mm512_sub_epi16(z9, z20); + z8 = _mm512_add_epi16(z2, z21); + z9 = _mm512_add_epi16(z3, z22); + z21 = _mm512_sub_epi16(z2, z21); + z22 = _mm512_sub_epi16(z3, z22); + z2 = _mm512_mulhi_epi16(z8, z15); + z3 = _mm512_mulhi_epi16(z9, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z8, z2); + z3 = _mm512_sub_epi16(z9, z3); + z8 = _mm512_mullo_epi16(z21, z12); + z9 = _mm512_mullo_epi16(z22, z12); + z21 = _mm512_mulhi_epi16(z21, z10); + z22 = _mm512_mulhi_epi16(z22, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z21 = _mm512_sub_epi16(z21, z8); + z22 = _mm512_sub_epi16(z22, z9); + z8 = _mm512_sub_epi16(z4, z23); + z9 = _mm512_sub_epi16(z5, z24); + z4 = _mm512_add_epi16(z4, z23); + z5 = _mm512_add_epi16(z5, z24); + z23 = _mm512_mullo_epi16(z8, z12); + z24 = _mm512_mullo_epi16(z9, z12); + z8 = _mm512_mulhi_epi16(z8, z10); + z9 = _mm512_mulhi_epi16(z9, z10); + z23 = _mm512_mulhi_epi16(z23, z14); + z24 = _mm512_mulhi_epi16(z24, z14); + z23 = _mm512_sub_epi16(z8, z23); + z24 = _mm512_sub_epi16(z9, z24); + z8 = _mm512_add_epi16(z6, z25); + z9 = _mm512_add_epi16(z7, z26); + z25 = _mm512_sub_epi16(z6, z25); + z26 = _mm512_sub_epi16(z7, z26); + z6 = _mm512_mulhi_epi16(z8, z15); + z7 = _mm512_mulhi_epi16(z9, z15); + z6 = _mm512_srai_epi16(z6, 10); + z7 = _mm512_srai_epi16(z7, 10); + z6 = _mm512_mullo_epi16(z6, z14); + z7 = _mm512_mullo_epi16(z7, z14); + z6 = _mm512_sub_epi16(z8, z6); + z7 = _mm512_sub_epi16(z9, z7); + z8 = _mm512_mullo_epi16(z25, z12); + z9 = _mm512_mullo_epi16(z26, z12); + z25 = _mm512_mulhi_epi16(z25, z10); + z26 = _mm512_mulhi_epi16(z26, z10); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z25 = _mm512_sub_epi16(z25, z8); + z26 = _mm512_sub_epi16(z26, z9); + z8 = _mm512_mullo_epi16(z0, z13); + z9 = _mm512_mullo_epi16(z1, z13); + z0 = _mm512_mulhi_epi16(z0, z11); + z1 = _mm512_mulhi_epi16(z1, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z0 = _mm512_sub_epi16(z0, z8); + z1 = _mm512_sub_epi16(z1, z9); + z8 = _mm512_mullo_epi16(z2, z13); + z9 = _mm512_mullo_epi16(z3, z13); + z2 = _mm512_mulhi_epi16(z2, z11); + z3 = _mm512_mulhi_epi16(z3, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z2 = _mm512_sub_epi16(z2, z8); + z3 = _mm512_sub_epi16(z3, z9); + z8 = _mm512_mullo_epi16(z4, z13); + z9 = _mm512_mullo_epi16(z5, z13); + z4 = _mm512_mulhi_epi16(z4, z11); + z5 = _mm512_mulhi_epi16(z5, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z4 = _mm512_sub_epi16(z4, z8); + z5 = _mm512_sub_epi16(z5, z9); + z8 = _mm512_mullo_epi16(z6, z13); + z9 = _mm512_mullo_epi16(z7, z13); + z6 = _mm512_mulhi_epi16(z6, z11); + z7 = _mm512_mulhi_epi16(z7, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z6 = _mm512_sub_epi16(z6, z8); + z7 = _mm512_sub_epi16(z7, z9); + z8 = _mm512_mullo_epi16(z19, z13); + z9 = _mm512_mullo_epi16(z20, z13); + z19 = _mm512_mulhi_epi16(z19, z11); + z20 = _mm512_mulhi_epi16(z20, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z19 = _mm512_sub_epi16(z19, z8); + z20 = _mm512_sub_epi16(z20, z9); + z8 = _mm512_mullo_epi16(z21, z13); + z9 = _mm512_mullo_epi16(z22, z13); + z21 = _mm512_mulhi_epi16(z21, z11); + z22 = _mm512_mulhi_epi16(z22, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z21 = _mm512_sub_epi16(z21, z8); + z22 = _mm512_sub_epi16(z22, z9); + z8 = _mm512_mullo_epi16(z23, z13); + z9 = _mm512_mullo_epi16(z24, z13); + z23 = _mm512_mulhi_epi16(z23, z11); + z24 = _mm512_mulhi_epi16(z24, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z23 = _mm512_sub_epi16(z23, z8); + z24 = _mm512_sub_epi16(z24, z9); + z8 = _mm512_mullo_epi16(z25, z13); + z9 = _mm512_mullo_epi16(z26, z13); + z25 = _mm512_mulhi_epi16(z25, z11); + z26 = _mm512_mulhi_epi16(z26, z11); + z8 = _mm512_mulhi_epi16(z8, z14); + z9 = _mm512_mulhi_epi16(z9, z14); + z25 = _mm512_sub_epi16(z25, z8); + z26 = _mm512_sub_epi16(z26, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_extracti64x4_epi64(z1, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_castsi512_si256(z2)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_extracti64x4_epi64(z2, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_castsi512_si256(z3)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_extracti64x4_epi64(z3, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z4)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_extracti64x4_epi64(z4, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_castsi512_si256(z5)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 160), _mm512_extracti64x4_epi64(z5, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_castsi512_si256(z6)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 192), _mm512_extracti64x4_epi64(z6, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_castsi512_si256(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 224), _mm512_extracti64x4_epi64(z7, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_castsi512_si256(z19)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 256), _mm512_extracti64x4_epi64( + z19, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_castsi512_si256(z20)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 288), _mm512_extracti64x4_epi64( + z20, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_castsi512_si256(z21)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 320), _mm512_extracti64x4_epi64( + z21, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_castsi512_si256(z22)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 352), _mm512_extracti64x4_epi64( + z22, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_castsi512_si256(z23)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 384), _mm512_extracti64x4_epi64( + z23, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_castsi512_si256(z24)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 416), _mm512_extracti64x4_epi64( + z24, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_castsi512_si256(z25)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 448), _mm512_extracti64x4_epi64( + z25, 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_castsi512_si256(z26)); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 480), _mm512_extracti64x4_epi64( + z26, 1)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rcx, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rcx, 64)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rcx, 128)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rcx, 192)); + z4 = _mm512_sub_epi16(z4, z0); + z5 = _mm512_sub_epi16(z5, z1); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_sub_epi16(z6, z2); + z7 = _mm512_sub_epi16(z7, z3); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rsi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rsi, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rcx, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rcx, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rcx, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rcx, 448)); + z4 = _mm512_sub_epi16(z4, z0); + z5 = _mm512_sub_epi16(z5, z1); + z0 = _mm512_mulhi_epi16(z4, z15); + z1 = _mm512_mulhi_epi16(z5, z15); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z14); + z1 = _mm512_mullo_epi16(z1, z14); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_sub_epi16(z6, z2); + z7 = _mm512_sub_epi16(z7, z3); + z2 = _mm512_mulhi_epi16(z6, z15); + z3 = _mm512_mulhi_epi16(z7, z15); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z14); + z3 = _mm512_mullo_epi16(z3, z14); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rsi, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rsi, 320), z1); + _mm512_storeu_si512((void*)WC_PW(rsi, 384), z2); + _mm512_storeu_si512((void*)WC_PW(rsi, 448), z3); +} + +#ifdef WOLFSSL_MLKEM_ASM_TEST +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_basemul_x2_avx512(sword16* r, const sword16* a, + const sword16* b) +{ + word64 rdi, rsi, rdx, r8, rcx, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13; + __mmask32 k3, k4; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + + r8 = (word64)(0xd01); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r8)); + z12 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z12)); + r8 = (word64)(0xf301); + z13 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r8)); + z13 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z13)); + rcx = (word32)(0xaaaaaaaa); + k3 = (__mmask32)(word32)rcx; + rcx = (word32)(0x55555555); + k4 = (__mmask32)(word32)rcx; + rax = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_slli_epi32(z1, 16); + z9 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z8); + z1 = _mm512_mask_blend_epi16(k4, z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_slli_epi32(z1, 16); + z9 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z8); + z1 = _mm512_mask_blend_epi16(k4, z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_slli_epi32(z1, 16); + z9 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z8); + z1 = _mm512_mask_blend_epi16(k4, z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_slli_epi32(z1, 16); + z9 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z8); + z1 = _mm512_mask_blend_epi16(k4, z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), _mm512_extracti64x4_epi64(z1, + 1)); +} + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_tomont_x2_avx512(sword16* v) +{ + word64 rdi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z8, z9, z10; + + rdi = (word64)(size_t)v; + + rdx = (word64)(0x549); + z8 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z8 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z8)); + rdx = (word64)(0x5049); + z9 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z9 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z9)); + rdx = (word64)(0xd01); + z10 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z10 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z10)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_mullo_epi16(z0, z9); + z5 = _mm512_mulhi_epi16(z0, z8); + z4 = _mm512_mulhi_epi16(z4, z10); + z0 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z1, z9); + z5 = _mm512_mulhi_epi16(z1, z8); + z4 = _mm512_mulhi_epi16(z4, z10); + z1 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z2, z9); + z5 = _mm512_mulhi_epi16(z2, z8); + z4 = _mm512_mulhi_epi16(z4, z10); + z2 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z3, z9); + z5 = _mm512_mulhi_epi16(z3, z8); + z4 = _mm512_mulhi_epi16(z4, z10); + z3 = _mm512_sub_epi16(z5, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z4 = _mm512_mullo_epi16(z0, z9); + z5 = _mm512_mulhi_epi16(z0, z8); + z4 = _mm512_mulhi_epi16(z4, z10); + z0 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z1, z9); + z5 = _mm512_mulhi_epi16(z1, z8); + z4 = _mm512_mulhi_epi16(z4, z10); + z1 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z2, z9); + z5 = _mm512_mulhi_epi16(z2, z8); + z4 = _mm512_mulhi_epi16(z4, z10); + z2 = _mm512_sub_epi16(z5, z4); + z4 = _mm512_mullo_epi16(z3, z9); + z5 = _mm512_mulhi_epi16(z3, z8); + z4 = _mm512_mulhi_epi16(z4, z10); + z3 = _mm512_sub_epi16(z5, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z3); +} + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_adderr_x2_avx512(sword16* r, const sword16* e) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)e; + + rdx = (word64)(0x4ebf); + z8 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z8 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z8)); + rdx = (word64)(0xd01); + z9 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z9 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z9)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z8); + z1 = _mm512_mulhi_epi16(z5, z8); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z9); + z1 = _mm512_mullo_epi16(z1, z9); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z8); + z3 = _mm512_mulhi_epi16(z7, z8); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z9); + z3 = _mm512_mullo_epi16(z3, z9); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z4 = _mm512_add_epi16(z0, z4); + z5 = _mm512_add_epi16(z1, z5); + z0 = _mm512_mulhi_epi16(z4, z8); + z1 = _mm512_mulhi_epi16(z5, z8); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z9); + z1 = _mm512_mullo_epi16(z1, z9); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_add_epi16(z2, z6); + z7 = _mm512_add_epi16(z3, z7); + z2 = _mm512_mulhi_epi16(z6, z8); + z3 = _mm512_mulhi_epi16(z7, z8); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z9); + z3 = _mm512_mullo_epi16(z3, z9); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z3); +} + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_suberr_x2_avx512(sword16* r, const sword16* e) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)e; + + rdx = (word64)(0x4ebf); + z8 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z8 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z8)); + rdx = (word64)(0xd01); + z9 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z9 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z9)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z4 = _mm512_sub_epi16(z4, z0); + z5 = _mm512_sub_epi16(z5, z1); + z0 = _mm512_mulhi_epi16(z4, z8); + z1 = _mm512_mulhi_epi16(z5, z8); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z9); + z1 = _mm512_mullo_epi16(z1, z9); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_sub_epi16(z6, z2); + z7 = _mm512_sub_epi16(z7, z3); + z2 = _mm512_mulhi_epi16(z6, z8); + z3 = _mm512_mulhi_epi16(z7, z8); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z9); + z3 = _mm512_mullo_epi16(z3, z9); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z4 = _mm512_sub_epi16(z4, z0); + z5 = _mm512_sub_epi16(z5, z1); + z0 = _mm512_mulhi_epi16(z4, z8); + z1 = _mm512_mulhi_epi16(z5, z8); + z0 = _mm512_srai_epi16(z0, 10); + z1 = _mm512_srai_epi16(z1, 10); + z0 = _mm512_mullo_epi16(z0, z9); + z1 = _mm512_mullo_epi16(z1, z9); + z0 = _mm512_sub_epi16(z4, z0); + z1 = _mm512_sub_epi16(z5, z1); + z6 = _mm512_sub_epi16(z6, z2); + z7 = _mm512_sub_epi16(z7, z3); + z2 = _mm512_mulhi_epi16(z6, z8); + z3 = _mm512_mulhi_epi16(z7, z8); + z2 = _mm512_srai_epi16(z2, 10); + z3 = _mm512_srai_epi16(z3, 10); + z2 = _mm512_mullo_epi16(z2, z9); + z3 = _mm512_mullo_epi16(z3, z9); + z2 = _mm512_sub_epi16(z6, z2); + z3 = _mm512_sub_epi16(z7, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z3); +} + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_ptacc_x2_avx512(sword16* r, const sword16* a, + const sword16* b, int kp) +{ + word64 rdi, rsi, rdx, rcx, r9, r8, r10, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13; + __mmask32 k3, k4; + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)a; + rdx = (word64)(size_t)b; + rcx = (word64)(word32)kp; + + r9 = (word64)(0xd01); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r9)); + z12 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z12)); + r9 = (word64)(0xf301); + z13 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r9)); + z13 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z13)); + r8 = (word32)(0xaaaaaaaa); + k3 = (__mmask32)(word32)r8; + r8 = (word32)(0x55555555); + k4 = (__mmask32)(word32)r8; + r10 = (word64)((word64)(sword64)(sword32)(word32)rcx); + rax = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rsi = (word64)(rsi + 0x200); + rdx = (word64)(rdx + 0x200); + r10 = (word64)(r10 - 2); + if ((r10) == (0)) { + goto L_pointwise_acc_mont_end_ptacc_x2_avx512; + } +L_pointwise_acc_mont_start_ptacc_x2_avx512: + rax = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rsi = (word64)(rsi + 0x200); + rdx = (word64)(rdx + 0x200); + r10 = (word64)(r10 - 1); + if ((sword64)(r10) > (sword64)(0)) { + goto L_pointwise_acc_mont_start_ptacc_x2_avx512; + } +L_pointwise_acc_mont_end_ptacc_x2_avx512: + rax = (word64)((word64)(size_t)L_mlkem_avx512_zetas_basemul); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 0))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 64)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 32))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 96)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 0))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 64)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 32))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 96)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 0))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 64)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 32))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 96)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 0))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 64)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 32))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 96)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_slli_epi32(z1, 16); + z9 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z8); + z1 = _mm512_mask_blend_epi16(k4, z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 128))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 192)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 160))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 224)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 128))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 192)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 160))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 224)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 128))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 192)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 160))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 224)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 128))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 192)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 160))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 224)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_slli_epi32(z1, 16); + z9 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z8); + z1 = _mm512_mask_blend_epi16(k4, z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 256))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 320)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 288))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 352)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 256))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 320)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 288))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 352)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 256))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 320)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 288))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 352)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 256))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 320)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 288))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 352)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_slli_epi32(z1, 16); + z9 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z8); + z1 = _mm512_mask_blend_epi16(k4, z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), _mm512_extracti64x4_epi64(z1, + 1)); + z2 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 384))); + z2 = _mm512_inserti64x4(z2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 448)), 1); + z3 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 416))); + z3 = _mm512_inserti64x4(z3, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 480)), 1); + z8 = _mm512_slli_epi32(z3, 16); + z9 = _mm512_srli_epi32(z2, 16); + z2 = _mm512_mask_blend_epi16(k3, z2, z8); + z3 = _mm512_mask_blend_epi16(k4, z3, z9); + z4 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 384))); + z4 = _mm512_inserti64x4(z4, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 448)), 1); + z5 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 416))); + z5 = _mm512_inserti64x4(z5, _mm256_loadu_si256((const __m256i*)WC_PR(rdx, + 480)), 1); + z8 = _mm512_slli_epi32(z5, 16); + z9 = _mm512_srli_epi32(z4, 16); + z4 = _mm512_mask_blend_epi16(k3, z4, z8); + z5 = _mm512_mask_blend_epi16(k4, z5, z9); + z6 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 384))); + z6 = _mm512_inserti64x4(z6, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 448)), 1); + z7 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rax, + 416))); + z7 = _mm512_inserti64x4(z7, _mm256_loadu_si256((const __m256i*)WC_PR(rax, + 480)), 1); + z0 = _mm512_mullo_epi16(z3, z5); + z8 = _mm512_mulhi_epi16(z3, z5); + z1 = _mm512_mullo_epi16(z2, z4); + z9 = _mm512_mulhi_epi16(z2, z4); + z10 = _mm512_mullo_epi16(z0, z13); + z11 = _mm512_mullo_epi16(z1, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z0 = _mm512_sub_epi16(z8, z10); + z1 = _mm512_sub_epi16(z9, z11); + z8 = _mm512_mullo_epi16(z0, z7); + z9 = _mm512_mulhi_epi16(z0, z6); + z8 = _mm512_mulhi_epi16(z8, z12); + z0 = _mm512_sub_epi16(z9, z8); + z0 = _mm512_add_epi16(z0, z1); + z1 = _mm512_mullo_epi16(z2, z5); + z8 = _mm512_mulhi_epi16(z2, z5); + z2 = _mm512_mullo_epi16(z3, z4); + z9 = _mm512_mulhi_epi16(z3, z4); + z10 = _mm512_mullo_epi16(z1, z13); + z11 = _mm512_mullo_epi16(z2, z13); + z10 = _mm512_mulhi_epi16(z10, z12); + z11 = _mm512_mulhi_epi16(z11, z12); + z1 = _mm512_sub_epi16(z8, z10); + z2 = _mm512_sub_epi16(z9, z11); + z1 = _mm512_add_epi16(z1, z2); + z8 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 384))); + z8 = _mm512_inserti64x4(z8, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 448)), 1); + z9 = _mm512_zextsi256_si512(_mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 416))); + z9 = _mm512_inserti64x4(z9, _mm256_loadu_si256((const __m256i*)WC_PR(rdi, + 480)), 1); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z8 = _mm512_slli_epi32(z1, 16); + z9 = _mm512_srli_epi32(z0, 16); + z0 = _mm512_mask_blend_epi16(k3, z0, z8); + z1 = _mm512_mask_blend_epi16(k4, z1, z9); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), _mm512_castsi512_si256(z0)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), _mm512_extracti64x4_epi64(z0, + 1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), _mm512_castsi512_si256(z1)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), _mm512_extracti64x4_epi64(z1, + 1)); + rsi = (word64)(rsi + 0x200); +} + +#endif /* WOLFSSL_MLKEM_ASM_TEST */ +XALIGNED(32) static const word64 L_mlkem_cbd_eta2_avx512_m55[] + WC_X64I_UNUSED = { + 0x5555555555555555ULL, 0x5555555555555555ULL, + 0x5555555555555555ULL, 0x5555555555555555ULL, + 0x5555555555555555ULL, 0x5555555555555555ULL, + 0x5555555555555555ULL, 0x5555555555555555ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_cbd_eta2_avx512_m33[] + WC_X64I_UNUSED = { + 0x3333333333333333ULL, 0x3333333333333333ULL, + 0x3333333333333333ULL, 0x3333333333333333ULL, + 0x3333333333333333ULL, 0x3333333333333333ULL, + 0x3333333333333333ULL, 0x3333333333333333ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_cbd_eta2_avx512_m03[] + WC_X64I_UNUSED = { + 0x0303030303030303ULL, 0x0303030303030303ULL, + 0x0303030303030303ULL, 0x0303030303030303ULL, + 0x0303030303030303ULL, 0x0303030303030303ULL, + 0x0303030303030303ULL, 0x0303030303030303ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_cbd_eta2_avx512_m0f[] + WC_X64I_UNUSED = { + 0x0f0f0f0f0f0f0f0fULL, 0x0f0f0f0f0f0f0f0fULL, + 0x0f0f0f0f0f0f0f0fULL, 0x0f0f0f0f0f0f0f0fULL, + 0x0f0f0f0f0f0f0f0fULL, 0x0f0f0f0f0f0f0f0fULL, + 0x0f0f0f0f0f0f0f0fULL, 0x0f0f0f0f0f0f0f0fULL, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_cbd_eta2_avx512(sword16* p, const byte* r) +{ + word64 rdi, rsi; + __m128i x11; + __m256i y6, y7, y8, y9, y10; + __m512i z0, z1, z2, z3, z4, z5; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + + z0 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta2_avx512_m55, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta2_avx512_m33, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta2_avx512_m03, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta2_avx512_m0f, 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z5 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z5 = _mm512_and_si512(z5, z0); + z4 = _mm512_add_epi8(z4, z5); + z5 = _mm512_srli_epi16(z4, 2); + z4 = _mm512_and_si512(z4, z1); + z5 = _mm512_and_si512(z5, z1); + z4 = _mm512_add_epi8(z4, z1); + z4 = _mm512_sub_epi8(z4, z5); + z5 = _mm512_srli_epi16(z4, 4); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_and_si512(z5, z3); + z4 = _mm512_sub_epi8(z4, z2); + z5 = _mm512_sub_epi8(z5, z2); + y6 = _mm256_unpacklo_epi8(_mm512_castsi512_si256(z4), + _mm512_castsi512_si256(z5)); + y7 = _mm256_unpackhi_epi8(_mm512_castsi512_si256(z4), + _mm512_castsi512_si256(z5)); + y10 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y6)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y10); + y10 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y10); + x11 = _mm256_extracti128_si256(y6, 1); + y10 = _mm256_cvtepi8_epi16(x11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y10); + x11 = _mm256_extracti128_si256(y7, 1); + y10 = _mm256_cvtepi8_epi16(x11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y10); + y8 = _mm512_extracti64x4_epi64(z4, 1); + y9 = _mm512_extracti64x4_epi64(z5, 1); + y6 = _mm256_unpacklo_epi8(y8, y9); + y7 = _mm256_unpackhi_epi8(y8, y9); + y10 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y6)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y10); + y10 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y10); + x11 = _mm256_extracti128_si256(y6, 1); + y10 = _mm256_cvtepi8_epi16(x11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y10); + x11 = _mm256_extracti128_si256(y7, 1); + y10 = _mm256_cvtepi8_epi16(x11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y10); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z5 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z5 = _mm512_and_si512(z5, z0); + z4 = _mm512_add_epi8(z4, z5); + z5 = _mm512_srli_epi16(z4, 2); + z4 = _mm512_and_si512(z4, z1); + z5 = _mm512_and_si512(z5, z1); + z4 = _mm512_add_epi8(z4, z1); + z4 = _mm512_sub_epi8(z4, z5); + z5 = _mm512_srli_epi16(z4, 4); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_and_si512(z5, z3); + z4 = _mm512_sub_epi8(z4, z2); + z5 = _mm512_sub_epi8(z5, z2); + y6 = _mm256_unpacklo_epi8(_mm512_castsi512_si256(z4), + _mm512_castsi512_si256(z5)); + y7 = _mm256_unpackhi_epi8(_mm512_castsi512_si256(z4), + _mm512_castsi512_si256(z5)); + y10 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y6)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y10); + y10 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y10); + x11 = _mm256_extracti128_si256(y6, 1); + y10 = _mm256_cvtepi8_epi16(x11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y10); + x11 = _mm256_extracti128_si256(y7, 1); + y10 = _mm256_cvtepi8_epi16(x11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y10); + y8 = _mm512_extracti64x4_epi64(z4, 1); + y9 = _mm512_extracti64x4_epi64(z5, 1); + y6 = _mm256_unpacklo_epi8(y8, y9); + y7 = _mm256_unpackhi_epi8(y8, y9); + y10 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y6)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y10); + y10 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(y7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y10); + x11 = _mm256_extracti128_si256(y6, 1); + y10 = _mm256_cvtepi8_epi16(x11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y10); + x11 = _mm256_extracti128_si256(y7, 1); + y10 = _mm256_cvtepi8_epi16(x11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y10); +} + +XALIGNED(32) static const word64 L_mlkem_cbd_eta3_avx512_m249[] + WC_X64I_UNUSED = { + 0x0024924900249249ULL, 0x0024924900249249ULL, + 0x0024924900249249ULL, 0x0024924900249249ULL, + 0x0024924900249249ULL, 0x0024924900249249ULL, + 0x0024924900249249ULL, 0x0024924900249249ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_cbd_eta3_avx512_m6db[] + WC_X64I_UNUSED = { + 0x006db6db006db6dbULL, 0x006db6db006db6dbULL, + 0x006db6db006db6dbULL, 0x006db6db006db6dbULL, + 0x006db6db006db6dbULL, 0x006db6db006db6dbULL, + 0x006db6db006db6dbULL, 0x006db6db006db6dbULL, +}; + +XALIGNED(32) static const word64 L_mlkem_cbd_eta3_avx512_m07[] + WC_X64I_UNUSED = { + 0x0000000700000007ULL, 0x0000000700000007ULL, + 0x0000000700000007ULL, 0x0000000700000007ULL, + 0x0000000700000007ULL, 0x0000000700000007ULL, + 0x0000000700000007ULL, 0x0000000700000007ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_cbd_eta3_avx512_m70[] + WC_X64I_UNUSED = { + 0x0007000000070000ULL, 0x0007000000070000ULL, + 0x0007000000070000ULL, 0x0007000000070000ULL, + 0x0007000000070000ULL, 0x0007000000070000ULL, + 0x0007000000070000ULL, 0x0007000000070000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_cbd_eta3_avx512_m3[] WC_X64I_UNUSED = { + 0x0003000300030003ULL, 0x0003000300030003ULL, + 0x0003000300030003ULL, 0x0003000300030003ULL, + 0x0003000300030003ULL, 0x0003000300030003ULL, + 0x0003000300030003ULL, 0x0003000300030003ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_cbd_eta3_avx512_shuff[] + WC_X64I_UNUSED = { + 0xff050403ff020100ULL, 0xff0b0a09ff080706ULL, + 0xff090807ff060504ULL, 0xff0f0e0dff0c0b0aULL, + 0xff050403ff020100ULL, 0xff0b0a09ff080706ULL, + 0xff090807ff060504ULL, 0xff0f0e0dff0c0b0aULL, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_cbd_eta3_avx512(sword16* p, const byte* r) +{ + word64 rdi, rsi; + __m256i y10, y11, y12, y13; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + + z0 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta3_avx512_m249, + 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta3_avx512_m6db, + 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta3_avx512_m07, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta3_avx512_m70, 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta3_avx512_m3, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_cbd_eta3_avx512_shuff, + 0)); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 0)), 0x94)); + y10 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 24)), 0x94); + z6 = _mm512_inserti64x4(z6, y10, 1); + z6 = _mm512_shuffle_epi8(z6, z5); + z7 = _mm512_srli_epi32(z6, 1); + z8 = _mm512_srli_epi32(z6, 2); + z6 = _mm512_and_si512(z6, z0); + z7 = _mm512_and_si512(z7, z0); + z8 = _mm512_and_si512(z8, z0); + z6 = _mm512_add_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z8); + z7 = _mm512_srli_epi32(z6, 3); + z6 = _mm512_add_epi32(z6, z1); + z6 = _mm512_sub_epi32(z6, z7); + z7 = _mm512_slli_epi32(z6, 10); + z8 = _mm512_srli_epi32(z6, 12); + z9 = _mm512_srli_epi32(z6, 2); + z6 = _mm512_and_si512(z6, z2); + z7 = _mm512_and_si512(z7, z3); + z8 = _mm512_and_si512(z8, z2); + z9 = _mm512_and_si512(z9, z3); + z6 = _mm512_add_epi16(z6, z7); + z7 = _mm512_add_epi16(z8, z9); + z6 = _mm512_sub_epi16(z6, z4); + z7 = _mm512_sub_epi16(z7, z4); + z8 = _mm512_zextsi256_si512(_mm256_unpacklo_epi32(_mm512_castsi512_si256( + z6), _mm512_castsi512_si256(z7))); + z9 = _mm512_zextsi256_si512(_mm256_unpackhi_epi32(_mm512_castsi512_si256( + z6), _mm512_castsi512_si256(z7))); + y12 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x20); + y13 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 32), y13); + y10 = _mm512_extracti64x4_epi64(z6, 1); + y11 = _mm512_extracti64x4_epi64(z7, 1); + z8 = _mm512_zextsi256_si512(_mm256_unpacklo_epi32(y10, y11)); + z9 = _mm512_zextsi256_si512(_mm256_unpackhi_epi32(y10, y11)); + y12 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x20); + y13 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 64), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 96), y13); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 48)), 0x94)); + y10 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 72)), 0x94); + z6 = _mm512_inserti64x4(z6, y10, 1); + z6 = _mm512_shuffle_epi8(z6, z5); + z7 = _mm512_srli_epi32(z6, 1); + z8 = _mm512_srli_epi32(z6, 2); + z6 = _mm512_and_si512(z6, z0); + z7 = _mm512_and_si512(z7, z0); + z8 = _mm512_and_si512(z8, z0); + z6 = _mm512_add_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z8); + z7 = _mm512_srli_epi32(z6, 3); + z6 = _mm512_add_epi32(z6, z1); + z6 = _mm512_sub_epi32(z6, z7); + z7 = _mm512_slli_epi32(z6, 10); + z8 = _mm512_srli_epi32(z6, 12); + z9 = _mm512_srli_epi32(z6, 2); + z6 = _mm512_and_si512(z6, z2); + z7 = _mm512_and_si512(z7, z3); + z8 = _mm512_and_si512(z8, z2); + z9 = _mm512_and_si512(z9, z3); + z6 = _mm512_add_epi16(z6, z7); + z7 = _mm512_add_epi16(z8, z9); + z6 = _mm512_sub_epi16(z6, z4); + z7 = _mm512_sub_epi16(z7, z4); + z8 = _mm512_zextsi256_si512(_mm256_unpacklo_epi32(_mm512_castsi512_si256( + z6), _mm512_castsi512_si256(z7))); + z9 = _mm512_zextsi256_si512(_mm256_unpackhi_epi32(_mm512_castsi512_si256( + z6), _mm512_castsi512_si256(z7))); + y12 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x20); + y13 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 128), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), y13); + y10 = _mm512_extracti64x4_epi64(z6, 1); + y11 = _mm512_extracti64x4_epi64(z7, 1); + z8 = _mm512_zextsi256_si512(_mm256_unpacklo_epi32(y10, y11)); + z9 = _mm512_zextsi256_si512(_mm256_unpackhi_epi32(y10, y11)); + y12 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x20); + y13 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 192), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 224), y13); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 96)), 0x94)); + y10 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 120)), 0x94); + z6 = _mm512_inserti64x4(z6, y10, 1); + z6 = _mm512_shuffle_epi8(z6, z5); + z7 = _mm512_srli_epi32(z6, 1); + z8 = _mm512_srli_epi32(z6, 2); + z6 = _mm512_and_si512(z6, z0); + z7 = _mm512_and_si512(z7, z0); + z8 = _mm512_and_si512(z8, z0); + z6 = _mm512_add_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z8); + z7 = _mm512_srli_epi32(z6, 3); + z6 = _mm512_add_epi32(z6, z1); + z6 = _mm512_sub_epi32(z6, z7); + z7 = _mm512_slli_epi32(z6, 10); + z8 = _mm512_srli_epi32(z6, 12); + z9 = _mm512_srli_epi32(z6, 2); + z6 = _mm512_and_si512(z6, z2); + z7 = _mm512_and_si512(z7, z3); + z8 = _mm512_and_si512(z8, z2); + z9 = _mm512_and_si512(z9, z3); + z6 = _mm512_add_epi16(z6, z7); + z7 = _mm512_add_epi16(z8, z9); + z6 = _mm512_sub_epi16(z6, z4); + z7 = _mm512_sub_epi16(z7, z4); + z8 = _mm512_zextsi256_si512(_mm256_unpacklo_epi32(_mm512_castsi512_si256( + z6), _mm512_castsi512_si256(z7))); + z9 = _mm512_zextsi256_si512(_mm256_unpackhi_epi32(_mm512_castsi512_si256( + z6), _mm512_castsi512_si256(z7))); + y12 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x20); + y13 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 256), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 288), y13); + y10 = _mm512_extracti64x4_epi64(z6, 1); + y11 = _mm512_extracti64x4_epi64(z7, 1); + z8 = _mm512_zextsi256_si512(_mm256_unpacklo_epi32(y10, y11)); + z9 = _mm512_zextsi256_si512(_mm256_unpackhi_epi32(y10, y11)); + y12 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x20); + y13 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 320), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 352), y13); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 144)), 0x94)); + y10 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 168)), 0x94); + z6 = _mm512_inserti64x4(z6, y10, 1); + z6 = _mm512_shuffle_epi8(z6, z5); + z7 = _mm512_srli_epi32(z6, 1); + z8 = _mm512_srli_epi32(z6, 2); + z6 = _mm512_and_si512(z6, z0); + z7 = _mm512_and_si512(z7, z0); + z8 = _mm512_and_si512(z8, z0); + z6 = _mm512_add_epi32(z6, z7); + z6 = _mm512_add_epi32(z6, z8); + z7 = _mm512_srli_epi32(z6, 3); + z6 = _mm512_add_epi32(z6, z1); + z6 = _mm512_sub_epi32(z6, z7); + z7 = _mm512_slli_epi32(z6, 10); + z8 = _mm512_srli_epi32(z6, 12); + z9 = _mm512_srli_epi32(z6, 2); + z6 = _mm512_and_si512(z6, z2); + z7 = _mm512_and_si512(z7, z3); + z8 = _mm512_and_si512(z8, z2); + z9 = _mm512_and_si512(z9, z3); + z6 = _mm512_add_epi16(z6, z7); + z7 = _mm512_add_epi16(z8, z9); + z6 = _mm512_sub_epi16(z6, z4); + z7 = _mm512_sub_epi16(z7, z4); + z8 = _mm512_zextsi256_si512(_mm256_unpacklo_epi32(_mm512_castsi512_si256( + z6), _mm512_castsi512_si256(z7))); + z9 = _mm512_zextsi256_si512(_mm256_unpackhi_epi32(_mm512_castsi512_si256( + z6), _mm512_castsi512_si256(z7))); + y12 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x20); + y13 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 384), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 416), y13); + y10 = _mm512_extracti64x4_epi64(z6, 1); + y11 = _mm512_extracti64x4_epi64(z7, 1); + z8 = _mm512_zextsi256_si512(_mm256_unpacklo_epi32(y10, y11)); + z9 = _mm512_zextsi256_si512(_mm256_unpackhi_epi32(y10, y11)); + y12 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x20); + y13 = _mm256_permute2x128_si256(_mm512_castsi512_si256(z8), + _mm512_castsi512_si256(z9), 0x31); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 448), y12); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 480), y13); +} + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_avx512_perm[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000002, 0x00000000, + 0x00000003, 0x00000004, 0x00000005, 0x00000000, + 0x00000006, 0x00000007, 0x00000008, 0x00000000, + 0x00000009, 0x0000000a, 0x0000000b, 0x00000000, +}; + +XALIGNED(16) static const byte L_mlkem_rej_uniform_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_avx512_mask[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +WC_X64I_TARGET("avx512f,avx512bw,bmi") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx512(sword16* p, + unsigned int len, const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, rbx = 0, r11 = 0, r8 = 0, r9 = 0; + __m256i y8 = _mm256_setzero_si256(); + __m512i z0, z1, z2, z3, z4 = _mm512_setzero_si512(), + z5 = _mm512_setzero_si512(), z6 = _mm512_setzero_si512(), + z7 = _mm512_setzero_si512(); + __mmask32 k1, k2; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + rcx = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + r10 = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r10)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_rej_uniform_avx512_perm, + 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_rej_uniform_avx512_shuf, + 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_rej_uniform_avx512_mask, + 0)); +L_mlkem_rej_uniform_avx512_fast: + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_avx512_fast_end; + } + if ((sword32)((word32)rcx) < (sword32)(0x40)) { + goto L_mlkem_rej_uniform_avx512_fast_end; + } + z4 = _mm512_permutexvar_epi32(z1, _mm512_loadu_si512((const void*)WC_PR(rdx, + 0))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + k1 = _mm512_cmp_epi16_mask(z4, z0, 1); + rbx = (word32)((word32)k1); + z6 = _mm512_cvtepu16_epi32(_mm512_castsi512_si256(z4)); + z7 = _mm512_mask_compress_epi32(z7, k1, z6); + z6 = _mm512_zextsi256_si512(_mm512_cvtepi32_epi16(z7)); + r11 = (word32)((word32)rbx); + r11 = (word32)((word32)r11 & 0xffff); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), _mm512_castsi512_si256(z6)); + y8 = _mm512_extracti64x4_epi64(z4, 1); + z6 = _mm512_cvtepu16_epi32(y8); + k2 = _kshiftri_mask32(k1, 16); + z7 = _mm512_mask_compress_epi32(z7, k2, z6); + z6 = _mm512_zextsi256_si512(_mm512_cvtepi32_epi16(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, r11 * 2), _mm512_castsi512_si256( + z6)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + rcx = (word32)((word32)rcx - 0x30); + goto L_mlkem_rej_uniform_avx512_fast; +L_mlkem_rej_uniform_avx512_fast_end: +L_mlkem_rej_uniform_avx512_tail: + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_tail_end; + } + if ((sword32)((word32)rcx) < (sword32)(3)) { + goto L_mlkem_rej_uniform_avx512_tail_end; + } + r8 = (word32)((word32)WC_L16(rdx, 0)); + r8 = (word32)((word32)r8 & 0xfff); + r9 = (word32)((word32)WC_L16(rdx, 1)); + r9 = (word32)((word32)r9 >> 4); + if ((sword32)((word32)r8) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_avx512_skip0; + } + WC_S16(rdi, 0) = (word16)((word16)r8); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_tail_end; + } +L_mlkem_rej_uniform_avx512_skip0: + if ((sword32)((word32)r9) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_avx512_skip1; + } + WC_S16(rdi, 0) = (word16)((word16)r9); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_tail_end; + } +L_mlkem_rej_uniform_avx512_skip1: + rdx = (word64)(rdx + 3); + rcx = (word32)((word32)rcx - 3); + goto L_mlkem_rej_uniform_avx512_tail; +L_mlkem_rej_uniform_avx512_tail_end: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; +} + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_n_avx512_perm[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000002, 0x00000000, + 0x00000003, 0x00000004, 0x00000005, 0x00000000, + 0x00000006, 0x00000007, 0x00000008, 0x00000000, + 0x00000009, 0x0000000a, 0x0000000b, 0x00000000, +}; + +XALIGNED(16) static const byte L_mlkem_rej_uniform_n_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_n_avx512_mask[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +WC_X64I_TARGET("avx512f,avx512bw,bmi") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx512(sword16* p, + unsigned int len, const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, rbx = 0, r11 = 0, r8 = 0, r9 = 0; + __m256i y8 = _mm256_setzero_si256(); + __m512i z0, z1, z2, z3, z4 = _mm512_setzero_si512(), + z5 = _mm512_setzero_si512(), z6 = _mm512_setzero_si512(), + z7 = _mm512_setzero_si512(); + __mmask32 k1, k2; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + rcx = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + r10 = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r10)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_perm, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_shuf, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_mask, 0)); +L_mlkem_rej_uniform_n_avx512_fast: + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_n_avx512_fast_end; + } + if ((sword32)((word32)rcx) < (sword32)(0x40)) { + goto L_mlkem_rej_uniform_n_avx512_fast_end; + } + z4 = _mm512_permutexvar_epi32(z1, _mm512_loadu_si512((const void*)WC_PR(rdx, + 0))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + k1 = _mm512_cmp_epi16_mask(z4, z0, 1); + rbx = (word32)((word32)k1); + z6 = _mm512_cvtepu16_epi32(_mm512_castsi512_si256(z4)); + z7 = _mm512_mask_compress_epi32(z7, k1, z6); + z6 = _mm512_zextsi256_si512(_mm512_cvtepi32_epi16(z7)); + r11 = (word32)((word32)rbx); + r11 = (word32)((word32)r11 & 0xffff); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), _mm512_castsi512_si256(z6)); + y8 = _mm512_extracti64x4_epi64(z4, 1); + z6 = _mm512_cvtepu16_epi32(y8); + k2 = _kshiftri_mask32(k1, 16); + z7 = _mm512_mask_compress_epi32(z7, k2, z6); + z6 = _mm512_zextsi256_si512(_mm512_cvtepi32_epi16(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, r11 * 2), _mm512_castsi512_si256( + z6)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + rcx = (word32)((word32)rcx - 0x30); + goto L_mlkem_rej_uniform_n_avx512_fast; +L_mlkem_rej_uniform_n_avx512_fast_end: +L_mlkem_rej_uniform_n_avx512_tail: + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_tail_end; + } + if ((sword32)((word32)rcx) < (sword32)(3)) { + goto L_mlkem_rej_uniform_n_avx512_tail_end; + } + r8 = (word32)((word32)WC_L16(rdx, 0)); + r8 = (word32)((word32)r8 & 0xfff); + r9 = (word32)((word32)WC_L16(rdx, 1)); + r9 = (word32)((word32)r9 >> 4); + if ((sword32)((word32)r8) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_n_avx512_skip0; + } + WC_S16(rdi, 0) = (word16)((word16)r8); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_tail_end; + } +L_mlkem_rej_uniform_n_avx512_skip0: + if ((sword32)((word32)r9) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_n_avx512_skip1; + } + WC_S16(rdi, 0) = (word16)((word16)r9); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_tail_end; + } +L_mlkem_rej_uniform_n_avx512_skip1: + rdx = (word64)(rdx + 3); + rcx = (word32)((word32)rcx - 3); + goto L_mlkem_rej_uniform_n_avx512_tail; +L_mlkem_rej_uniform_n_avx512_tail_end: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; +} + +XALIGNED(32) static const word64 L_mlkem_redistribute_8_rand_avx512_idx0[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, + 0x0000000000000008ULL, 0x000000000000000cULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_8_rand_avx512_idx1[] + WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000005ULL, + 0x0000000000000009ULL, 0x000000000000000dULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_8_rand_avx512_idx2[] + WC_X64I_UNUSED = { + 0x0000000000000002ULL, 0x0000000000000006ULL, + 0x000000000000000aULL, 0x000000000000000eULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_8_rand_avx512_idx3[] + WC_X64I_UNUSED = { + 0x0000000000000003ULL, 0x0000000000000007ULL, + 0x000000000000000bULL, 0x000000000000000fULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void mlkem_redistribute_8_rand_avx512(const word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8; + __m512i z0, z1, z2; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_8_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_8_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_8_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_8_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_8_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_8_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_8_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_8_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), _mm512_castsi512_si256(z2)); +} + +XALIGNED(32) static const word64 L_mlkem_redistribute_16_rand_avx512_idx0[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, + 0x0000000000000008ULL, 0x000000000000000cULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_16_rand_avx512_idx1[] + WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000005ULL, + 0x0000000000000009ULL, 0x000000000000000dULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_16_rand_avx512_idx2[] + WC_X64I_UNUSED = { + 0x0000000000000002ULL, 0x0000000000000006ULL, + 0x000000000000000aULL, 0x000000000000000eULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_16_rand_avx512_idx3[] + WC_X64I_UNUSED = { + 0x0000000000000003ULL, 0x0000000000000007ULL, + 0x000000000000000bULL, 0x000000000000000fULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void mlkem_redistribute_16_rand_avx512(const word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8; + __m512i z0, z1, z2; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_16_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), _mm512_castsi512_si256(z2)); +} + +XALIGNED(32) static const word64 L_mlkem_redistribute_17_rand_avx512_idx0[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, + 0x0000000000000008ULL, 0x000000000000000cULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_17_rand_avx512_idx1[] + WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000005ULL, + 0x0000000000000009ULL, 0x000000000000000dULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_17_rand_avx512_idx2[] + WC_X64I_UNUSED = { + 0x0000000000000002ULL, 0x0000000000000006ULL, + 0x000000000000000aULL, 0x000000000000000eULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_17_rand_avx512_idx3[] + WC_X64I_UNUSED = { + 0x0000000000000003ULL, 0x0000000000000007ULL, + 0x000000000000000bULL, 0x000000000000000fULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void mlkem_redistribute_17_rand_avx512(const word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11; + __m512i z0, z1, z2; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_17_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), _mm512_castsi512_si256(z2)); + rax = (word64)(WC_L64(rdi, 512)); + r9 = (word64)(WC_L64(rdi, 520)); + r10 = (word64)(WC_L64(rdi, 528)); + r11 = (word64)(WC_L64(rdi, 536)); + WC_S64(rsi, 128) = (word64)(rax); + WC_S64(rdx, 128) = (word64)(r9); + WC_S64(rcx, 128) = (word64)(r10); + WC_S64(r8, 128) = (word64)(r11); +} + +XALIGNED(32) static const word64 L_mlkem_redistribute_21_rand_avx512_idx0[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000004ULL, + 0x0000000000000008ULL, 0x000000000000000cULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_21_rand_avx512_idx1[] + WC_X64I_UNUSED = { + 0x0000000000000001ULL, 0x0000000000000005ULL, + 0x0000000000000009ULL, 0x000000000000000dULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_21_rand_avx512_idx2[] + WC_X64I_UNUSED = { + 0x0000000000000002ULL, 0x0000000000000006ULL, + 0x000000000000000aULL, 0x000000000000000eULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_redistribute_21_rand_avx512_idx3[] + WC_X64I_UNUSED = { + 0x0000000000000003ULL, 0x0000000000000007ULL, + 0x000000000000000bULL, 0x000000000000000fULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, +}; + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL void mlkem_redistribute_21_rand_avx512(const word64* s, byte* r0, + byte* r1, byte* r2, byte* r3) +{ + word64 rdi, rsi, rdx, rcx, r8, rax, r9, r10, r11; + __m512i z0, z1, z2; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)r0; + rdx = (word64)(size_t)r1; + rcx = (word64)(size_t)r2; + r8 = (word64)(size_t)r3; + + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 0), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 0), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 32), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 32), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 64), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 64), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 64), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 64), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 96), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 96), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 96), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 96), _mm512_castsi512_si256(z2)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx0, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rsi, 128), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx1, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rdx, 128), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx2, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(rcx, 128), _mm512_castsi512_si256(z2)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_redistribute_21_rand_avx512_idx3, 0)); + z2 = _mm512_permutex2var_epi64(z0, z2, z1); + _mm256_storeu_si256((__m256i*)WC_PW(r8, 128), _mm512_castsi512_si256(z2)); + rax = (word64)(WC_L64(rdi, 640)); + r9 = (word64)(WC_L64(rdi, 648)); + r10 = (word64)(WC_L64(rdi, 656)); + r11 = (word64)(WC_L64(rdi, 664)); + WC_S64(rsi, 160) = (word64)(rax); + WC_S64(rdx, 160) = (word64)(r9); + WC_S64(rcx, 160) = (word64)(r10); + WC_S64(r8, 160) = (word64)(r11); +} + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void mlkem_redistribute_8_rand_x8_avx512(const word64* s, + byte* out, word32 stride) +{ + word64 rdi, rsi, rdx, r13, rax, rcx, r8, r9, r10, r11, r12; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)out; + rdx = (word64)(word32)stride; + + r13 = (word32)((word32)rdx); + rax = (word64)(rsi); + rax = (word64)(rax + r13); + rcx = (word64)(rax); + rcx = (word64)(rcx + r13); + r8 = (word64)(rcx); + r8 = (word64)(r8 + r13); + r9 = (word64)(r8); + r9 = (word64)(r9 + r13); + r10 = (word64)(r9); + r10 = (word64)(r10 + r13); + r11 = (word64)(r10); + r11 = (word64)(r11 + r13); + r12 = (word64)(r11); + r12 = (word64)(r12 + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 0), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 0), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 0), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 0), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z7); +} + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void mlkem_redistribute_16_rand_x8_avx512(const word64* s, + byte* out, word32 stride) +{ + word64 rdi, rsi, rdx, r13, rax, rcx, r8, r9, r10, r11, r12; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)out; + rdx = (word64)(word32)stride; + + r13 = (word32)((word32)rdx); + rax = (word64)(rsi); + rax = (word64)(rax + r13); + rcx = (word64)(rax); + rcx = (word64)(rcx + r13); + r8 = (word64)(rcx); + r8 = (word64)(r8 + r13); + r9 = (word64)(r8); + r9 = (word64)(r9 + r13); + r10 = (word64)(r9); + r10 = (word64)(r10 + r13); + r11 = (word64)(r10); + r11 = (word64)(r11 + r13); + r12 = (word64)(r11); + r12 = (word64)(r12 + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 0), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 0), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 0), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 0), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z7); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 64), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 64), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 64), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 64), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z7); +} + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void mlkem_redistribute_17_rand_x8_avx512(const word64* s, + byte* out, word32 stride) +{ + word64 rdi, rsi, rdx, r13, rax, rcx, r8, r9, r10, r11, r12, r14; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)out; + rdx = (word64)(word32)stride; + + r13 = (word32)((word32)rdx); + rax = (word64)(rsi); + rax = (word64)(rax + r13); + rcx = (word64)(rax); + rcx = (word64)(rcx + r13); + r8 = (word64)(rcx); + r8 = (word64)(r8 + r13); + r9 = (word64)(r8); + r9 = (word64)(r9 + r13); + r10 = (word64)(r9); + r10 = (word64)(r10 + r13); + r11 = (word64)(r10); + r11 = (word64)(r11 + r13); + r12 = (word64)(r11); + r12 = (word64)(r12 + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 0), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 0), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 0), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 0), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z7); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 64), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 64), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 64), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 64), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z7); + r14 = (word64)(WC_L64(rdi, 1024)); + WC_S64(rsi, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1032)); + WC_S64(rax, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1040)); + WC_S64(rcx, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1048)); + WC_S64(r8, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1056)); + WC_S64(r9, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1064)); + WC_S64(r10, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1072)); + WC_S64(r11, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1080)); + WC_S64(r12, 128) = (word64)(r14); +} + +WC_X64I_TARGET("avx,avx512f") +WOLFSSL_LOCAL void mlkem_redistribute_21_rand_x8_avx512(const word64* s, + byte* out, word32 stride) +{ + word64 rdi, rsi, rdx, r13, rax, rcx, r8, r9, r10, r11, r12, r14; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, + z15, z16, z17, z18, z19, z20, z21, z22, z23; + + rdi = (word64)(size_t)s; + rsi = (word64)(size_t)out; + rdx = (word64)(word32)stride; + + r13 = (word32)((word32)rdx); + rax = (word64)(rsi); + rax = (word64)(rax + r13); + rcx = (word64)(rax); + rcx = (word64)(rcx + r13); + r8 = (word64)(rcx); + r8 = (word64)(r8 + r13); + r9 = (word64)(r8); + r9 = (word64)(r9 + r13); + r10 = (word64)(r9); + r10 = (word64)(r10 + r13); + r11 = (word64)(r10); + r11 = (word64)(r11 + r13); + r12 = (word64)(r11); + r12 = (word64)(r12 + r13); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 0), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 0), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 0), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 0), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 0), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 0), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 0), z7); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z8 = _mm512_unpacklo_epi64(z0, z1); + z9 = _mm512_unpackhi_epi64(z0, z1); + z10 = _mm512_unpacklo_epi64(z2, z3); + z11 = _mm512_unpackhi_epi64(z2, z3); + z12 = _mm512_unpacklo_epi64(z4, z5); + z13 = _mm512_unpackhi_epi64(z4, z5); + z14 = _mm512_unpacklo_epi64(z6, z7); + z15 = _mm512_unpackhi_epi64(z6, z7); + z16 = _mm512_shuffle_i64x2(z8, z10, 0x88); + z17 = _mm512_shuffle_i64x2(z9, z11, 0x88); + z18 = _mm512_shuffle_i64x2(z8, z10, 0xdd); + z19 = _mm512_shuffle_i64x2(z9, z11, 0xdd); + z20 = _mm512_shuffle_i64x2(z12, z14, 0x88); + z21 = _mm512_shuffle_i64x2(z13, z15, 0x88); + z22 = _mm512_shuffle_i64x2(z12, z14, 0xdd); + z23 = _mm512_shuffle_i64x2(z13, z15, 0xdd); + z0 = _mm512_shuffle_i64x2(z16, z20, 0x88); + z4 = _mm512_shuffle_i64x2(z16, z20, 0xdd); + z1 = _mm512_shuffle_i64x2(z17, z21, 0x88); + z5 = _mm512_shuffle_i64x2(z17, z21, 0xdd); + z2 = _mm512_shuffle_i64x2(z18, z22, 0x88); + z6 = _mm512_shuffle_i64x2(z18, z22, 0xdd); + z3 = _mm512_shuffle_i64x2(z19, z23, 0x88); + z7 = _mm512_shuffle_i64x2(z19, z23, 0xdd); + _mm512_storeu_si512((void*)WC_PW(rsi, 64), z0); + _mm512_storeu_si512((void*)WC_PW(rax, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rcx, 64), z2); + _mm512_storeu_si512((void*)WC_PW(r8, 64), z3); + _mm512_storeu_si512((void*)WC_PW(r9, 64), z4); + _mm512_storeu_si512((void*)WC_PW(r10, 64), z5); + _mm512_storeu_si512((void*)WC_PW(r11, 64), z6); + _mm512_storeu_si512((void*)WC_PW(r12, 64), z7); + r14 = (word64)(WC_L64(rdi, 1024)); + WC_S64(rsi, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1032)); + WC_S64(rax, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1040)); + WC_S64(rcx, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1048)); + WC_S64(r8, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1056)); + WC_S64(r9, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1064)); + WC_S64(r10, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1072)); + WC_S64(r11, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1080)); + WC_S64(r12, 128) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1088)); + WC_S64(rsi, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1096)); + WC_S64(rax, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1104)); + WC_S64(rcx, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1112)); + WC_S64(r8, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1120)); + WC_S64(r9, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1128)); + WC_S64(r10, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1136)); + WC_S64(r11, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1144)); + WC_S64(r12, 136) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1152)); + WC_S64(rsi, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1160)); + WC_S64(rax, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1168)); + WC_S64(rcx, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1176)); + WC_S64(r8, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1184)); + WC_S64(r9, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1192)); + WC_S64(r10, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1200)); + WC_S64(r11, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1208)); + WC_S64(r12, 144) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1216)); + WC_S64(rsi, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1224)); + WC_S64(rax, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1232)); + WC_S64(rcx, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1240)); + WC_S64(r8, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1248)); + WC_S64(r9, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1256)); + WC_S64(r10, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1264)); + WC_S64(r11, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1272)); + WC_S64(r12, 152) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1280)); + WC_S64(rsi, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1288)); + WC_S64(rax, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1296)); + WC_S64(rcx, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1304)); + WC_S64(r8, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1312)); + WC_S64(r9, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1320)); + WC_S64(r10, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1328)); + WC_S64(r11, 160) = (word64)(r14); + r14 = (word64)(WC_L64(rdi, 1336)); + WC_S64(r12, 160) = (word64)(r14); +} + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_csubq_avx512(sword16* p) +{ + word64 rdi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12; + + rdi = (word64)(size_t)p; + + rdx = (word64)(0xd01); + z12 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z12 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z12)); + z0 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z2 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z5 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z6 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z7 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z8 = _mm512_sub_epi16(z0, z12); + z9 = _mm512_sub_epi16(z1, z12); + z10 = _mm512_sub_epi16(z2, z12); + z11 = _mm512_sub_epi16(z3, z12); + z0 = _mm512_srai_epi16(z8, 15); + z1 = _mm512_srai_epi16(z9, 15); + z2 = _mm512_srai_epi16(z10, 15); + z3 = _mm512_srai_epi16(z11, 15); + z0 = _mm512_and_si512(z0, z12); + z1 = _mm512_and_si512(z1, z12); + z2 = _mm512_and_si512(z2, z12); + z3 = _mm512_and_si512(z3, z12); + z0 = _mm512_add_epi16(z0, z8); + z1 = _mm512_add_epi16(z1, z9); + z2 = _mm512_add_epi16(z2, z10); + z3 = _mm512_add_epi16(z3, z11); + z8 = _mm512_sub_epi16(z4, z12); + z9 = _mm512_sub_epi16(z5, z12); + z10 = _mm512_sub_epi16(z6, z12); + z11 = _mm512_sub_epi16(z7, z12); + z4 = _mm512_srai_epi16(z8, 15); + z5 = _mm512_srai_epi16(z9, 15); + z6 = _mm512_srai_epi16(z10, 15); + z7 = _mm512_srai_epi16(z11, 15); + z4 = _mm512_and_si512(z4, z12); + z5 = _mm512_and_si512(z5, z12); + z6 = _mm512_and_si512(z6, z12); + z7 = _mm512_and_si512(z7, z12); + z4 = _mm512_add_epi16(z4, z8); + z5 = _mm512_add_epi16(z5, z9); + z6 = _mm512_add_epi16(z6, z10); + z7 = _mm512_add_epi16(z7, z11); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); +} + +WC_X64I_TARGET("avx512f") +WOLFSSL_LOCAL int mlkem_cmp_avx512(const byte* a, const byte* b, int sz) +{ + word64 rdi, rsi, rdx, rcx, r8, rax = 0; + __m256i y2, y3 = _mm256_setzero_si256(); + __m512i z0, z1; + word32 zf1; + word32 zf2; + + rdi = (word64)(size_t)a; + rsi = (word64)(size_t)b; + rdx = (word64)(word32)sz; + + z0 = _mm512_setzero_si512(); + y2 = _mm256_setzero_si256(); + rcx = (word32)(0); + r8 = (word32)(-1); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 0)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 0)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 64)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 64)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 128)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 128)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 192)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 192)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 256)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 256)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 320)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 320)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 384)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 384)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 448)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 448)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 512)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 512)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 576)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 576)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 640)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 640)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 704)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 704)), 0xf6); + rdx = (word32)((word32)rdx - 0x300); + if (((word32)rdx) == (0)) { + goto L_mlkem_cmp_avx512_done; + } + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 768)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 768)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 832)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 832)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 896)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 896)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 960)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 960)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1024)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 1024)), 0xf6); + rdx = (word32)((word32)rdx - 0x140); + if (((word32)rdx) == (0)) { + goto L_mlkem_cmp_avx512_done; + } + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1088)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 1088)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1152)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 1152)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1216)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 1216)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1280)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 1280)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1344)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 1344)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1408)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 1408)), 0xf6); + z1 = _mm512_loadu_si512((const void*)WC_PR(rdi, 1472)); + z0 = _mm512_ternarylogic_epi32(z0, z1, _mm512_loadu_si512(( + const void*)WC_PR(rsi, 1472)), 0xf6); + y2 = _mm256_loadu_si256((const __m256i*)WC_PR(rdi, 1536)); + y2 = _mm256_xor_si256(y2, _mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 1536))); +L_mlkem_cmp_avx512_done: + y3 = _mm512_extracti64x4_epi64(z0, 1); + z0 = _mm512_zextsi256_si512(_mm256_or_si256(_mm512_castsi512_si256(z0), + y3)); + z0 = _mm512_zextsi256_si512(_mm256_or_si256(_mm512_castsi512_si256(z0), + y2)); + zf1 = _mm256_testz_si256(_mm512_castsi512_si256(z0), _mm512_castsi512_si256( + z0)); + zf2 = 1; + rax = (word32)((_mm256_testz_si256(_mm512_castsi512_si256(z0), + _mm512_castsi512_si256(z0))) == (1) ? (word32)rcx : (word32)rax); + rax = (word32)((zf1) != (zf2) ? (word32)r8 : (word32)rax); + return (int)(word32)rax; +} + +XALIGNED(16) static const word32 L_mlkem_from_bytes_avx512_perm_lo[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000002, 0x00000000, + 0x00000003, 0x00000004, 0x00000005, 0x00000000, + 0x00000006, 0x00000007, 0x00000008, 0x00000000, + 0x00000009, 0x0000000a, 0x0000000b, 0x00000000, +}; + +XALIGNED(16) static const word32 L_mlkem_from_bytes_avx512_perm_hi[] + WC_X64I_UNUSED = { + 0x00000004, 0x00000005, 0x00000006, 0x00000000, + 0x00000007, 0x00000008, 0x00000009, 0x00000000, + 0x0000000a, 0x0000000b, 0x0000000c, 0x00000000, + 0x0000000d, 0x0000000e, 0x0000000f, 0x00000000, +}; + +XALIGNED(16) static const byte L_mlkem_from_bytes_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, +}; + +XALIGNED(16) static const word32 L_mlkem_from_bytes_avx512_mask[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +WC_X64I_TARGET("avx,avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_from_bytes_avx512(sword16* p, const byte* b) +{ + word64 rdi, rsi; + __m512i z0, z1, z2, z3, z4, z5; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)b; + + z0 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_from_bytes_avx512_perm_lo, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_from_bytes_avx512_perm_hi, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_from_bytes_avx512_shuf, + 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_from_bytes_avx512_mask, + 0)); + z4 = _mm512_permutexvar_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 0))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z4); + z4 = _mm512_permutexvar_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 48))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z4); + z4 = _mm512_permutexvar_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 96))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z4); + z4 = _mm512_permutexvar_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 144))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z4); + z4 = _mm512_permutexvar_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 192))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + z4 = _mm512_permutexvar_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 240))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z4); + z4 = _mm512_permutexvar_epi32(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 288))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z4); + z4 = _mm512_permutexvar_epi32(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, + 320))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z4); +} + +XALIGNED(16) static const word32 L_mlkem_to_bytes_avx512_mask[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const byte L_mlkem_to_bytes_avx512_shuf[] WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, + 0x0a, 0x0c, 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, + 0x0a, 0x0c, 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, + 0x0a, 0x0c, 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, + 0x0a, 0x0c, 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(16) static const word32 L_mlkem_to_bytes_avx512_perm[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000002, 0x00000004, + 0x00000005, 0x00000006, 0x00000008, 0x00000009, + 0x0000000a, 0x0000000c, 0x0000000d, 0x0000000e, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_to_bytes_avx512(byte* b, sword16* p) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7; + + rdi = (word64)(size_t)b; + rsi = (word64)(size_t)p; + + rdx = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_to_bytes_avx512_mask, + 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_to_bytes_avx512_shuf, + 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_to_bytes_avx512_perm, + 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z5 = _mm512_sub_epi16(z4, z0); + z6 = _mm512_srai_epi16(z5, 15); + z6 = _mm512_and_si512(z6, z0); + z4 = _mm512_add_epi16(z6, z5); + z5 = _mm512_srli_epi32(z4, 16); + z5 = _mm512_slli_epi32(z5, 12); + z4 = _mm512_ternarylogic_epi32(z4, z1, z5, 0xea); + z4 = _mm512_shuffle_epi8(z4, z2); + z7 = _mm512_permutexvar_epi32(z3, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z7); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z5 = _mm512_sub_epi16(z4, z0); + z6 = _mm512_srai_epi16(z5, 15); + z6 = _mm512_and_si512(z6, z0); + z4 = _mm512_add_epi16(z6, z5); + z5 = _mm512_srli_epi32(z4, 16); + z5 = _mm512_slli_epi32(z5, 12); + z4 = _mm512_ternarylogic_epi32(z4, z1, z5, 0xea); + z4 = _mm512_shuffle_epi8(z4, z2); + z7 = _mm512_permutexvar_epi32(z3, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 48), z7); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z5 = _mm512_sub_epi16(z4, z0); + z6 = _mm512_srai_epi16(z5, 15); + z6 = _mm512_and_si512(z6, z0); + z4 = _mm512_add_epi16(z6, z5); + z5 = _mm512_srli_epi32(z4, 16); + z5 = _mm512_slli_epi32(z5, 12); + z4 = _mm512_ternarylogic_epi32(z4, z1, z5, 0xea); + z4 = _mm512_shuffle_epi8(z4, z2); + z7 = _mm512_permutexvar_epi32(z3, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 96), z7); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z5 = _mm512_sub_epi16(z4, z0); + z6 = _mm512_srai_epi16(z5, 15); + z6 = _mm512_and_si512(z6, z0); + z4 = _mm512_add_epi16(z6, z5); + z5 = _mm512_srli_epi32(z4, 16); + z5 = _mm512_slli_epi32(z5, 12); + z4 = _mm512_ternarylogic_epi32(z4, z1, z5, 0xea); + z4 = _mm512_shuffle_epi8(z4, z2); + z7 = _mm512_permutexvar_epi32(z3, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 144), z7); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z5 = _mm512_sub_epi16(z4, z0); + z6 = _mm512_srai_epi16(z5, 15); + z6 = _mm512_and_si512(z6, z0); + z4 = _mm512_add_epi16(z6, z5); + z5 = _mm512_srli_epi32(z4, 16); + z5 = _mm512_slli_epi32(z5, 12); + z4 = _mm512_ternarylogic_epi32(z4, z1, z5, 0xea); + z4 = _mm512_shuffle_epi8(z4, z2); + z7 = _mm512_permutexvar_epi32(z3, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z7); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z5 = _mm512_sub_epi16(z4, z0); + z6 = _mm512_srai_epi16(z5, 15); + z6 = _mm512_and_si512(z6, z0); + z4 = _mm512_add_epi16(z6, z5); + z5 = _mm512_srli_epi32(z4, 16); + z5 = _mm512_slli_epi32(z5, 12); + z4 = _mm512_ternarylogic_epi32(z4, z1, z5, 0xea); + z4 = _mm512_shuffle_epi8(z4, z2); + z7 = _mm512_permutexvar_epi32(z3, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 240), z7); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z5 = _mm512_sub_epi16(z4, z0); + z6 = _mm512_srai_epi16(z5, 15); + z6 = _mm512_and_si512(z6, z0); + z4 = _mm512_add_epi16(z6, z5); + z5 = _mm512_srli_epi32(z4, 16); + z5 = _mm512_slli_epi32(z5, 12); + z4 = _mm512_ternarylogic_epi32(z4, z1, z5, 0xea); + z4 = _mm512_shuffle_epi8(z4, z2); + z7 = _mm512_permutexvar_epi32(z3, z4); + _mm512_storeu_si512((void*)WC_PW(rdi, 288), z7); + z4 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z5 = _mm512_sub_epi16(z4, z0); + z6 = _mm512_srai_epi16(z5, 15); + z6 = _mm512_and_si512(z6, z0); + z4 = _mm512_add_epi16(z6, z5); + z5 = _mm512_srli_epi32(z4, 16); + z5 = _mm512_slli_epi32(z5, 12); + z4 = _mm512_ternarylogic_epi32(z4, z1, z5, 0xea); + z4 = _mm512_shuffle_epi8(z4, z2); + z7 = _mm512_permutexvar_epi32(z3, z4); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 336), _mm512_castsi512_si256(z7)); + z5 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z7, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 368), _mm512_castsi512_si128(z5)); +} + +XALIGNED(16) static const byte L_mlkem_from_msg_avx512_shuf[] WC_X64I_UNUSED = { + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x01, 0x80, 0x01, 0x80, 0x01, 0x80, 0x01, 0x80, + 0x01, 0x80, 0x01, 0x80, 0x01, 0x80, 0x01, 0x80, + 0x02, 0x80, 0x02, 0x80, 0x02, 0x80, 0x02, 0x80, + 0x02, 0x80, 0x02, 0x80, 0x02, 0x80, 0x02, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, +}; + +XALIGNED(16) static const word16 L_mlkem_from_msg_avx512_pow[] + WC_X64I_UNUSED = { + 0x8000, 0x4000, 0x2000, 0x1000, 0x0800, 0x0400, 0x0200, 0x0100, + 0x8000, 0x4000, 0x2000, 0x1000, 0x0800, 0x0400, 0x0200, 0x0100, + 0x8000, 0x4000, 0x2000, 0x1000, 0x0800, 0x0400, 0x0200, 0x0100, + 0x8000, 0x4000, 0x2000, 0x1000, 0x0800, 0x0400, 0x0200, 0x0100, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_from_msg_avx512(sword16* p, const byte* msg) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)msg; + + z0 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_from_msg_avx512_shuf, + 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_from_msg_avx512_pow, 0)); + rdx = (word64)(0x681); + z2 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z2 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z2)); + z3 = _mm512_set1_epi32((int)WC_L32(rsi, 0)); + z3 = _mm512_shuffle_epi8(z3, z0); + z3 = _mm512_mullo_epi16(z3, z1); + z3 = _mm512_srai_epi16(z3, 15); + z3 = _mm512_and_si512(z3, z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z3); + z3 = _mm512_set1_epi32((int)WC_L32(rsi, 4)); + z3 = _mm512_shuffle_epi8(z3, z0); + z3 = _mm512_mullo_epi16(z3, z1); + z3 = _mm512_srai_epi16(z3, 15); + z3 = _mm512_and_si512(z3, z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z3); + z3 = _mm512_set1_epi32((int)WC_L32(rsi, 8)); + z3 = _mm512_shuffle_epi8(z3, z0); + z3 = _mm512_mullo_epi16(z3, z1); + z3 = _mm512_srai_epi16(z3, 15); + z3 = _mm512_and_si512(z3, z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z3); + z3 = _mm512_set1_epi32((int)WC_L32(rsi, 12)); + z3 = _mm512_shuffle_epi8(z3, z0); + z3 = _mm512_mullo_epi16(z3, z1); + z3 = _mm512_srai_epi16(z3, 15); + z3 = _mm512_and_si512(z3, z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z3); + z3 = _mm512_set1_epi32((int)WC_L32(rsi, 16)); + z3 = _mm512_shuffle_epi8(z3, z0); + z3 = _mm512_mullo_epi16(z3, z1); + z3 = _mm512_srai_epi16(z3, 15); + z3 = _mm512_and_si512(z3, z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z3); + z3 = _mm512_set1_epi32((int)WC_L32(rsi, 20)); + z3 = _mm512_shuffle_epi8(z3, z0); + z3 = _mm512_mullo_epi16(z3, z1); + z3 = _mm512_srai_epi16(z3, 15); + z3 = _mm512_and_si512(z3, z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z3); + z3 = _mm512_set1_epi32((int)WC_L32(rsi, 24)); + z3 = _mm512_shuffle_epi8(z3, z0); + z3 = _mm512_mullo_epi16(z3, z1); + z3 = _mm512_srai_epi16(z3, 15); + z3 = _mm512_and_si512(z3, z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z3); + z3 = _mm512_set1_epi32((int)WC_L32(rsi, 28)); + z3 = _mm512_shuffle_epi8(z3, z0); + z3 = _mm512_mullo_epi16(z3, z1); + z3 = _mm512_srai_epi16(z3, 15); + z3 = _mm512_and_si512(z3, z2); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z3); +} + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_to_msg_avx512(byte* m, sword16* p) +{ + word64 rdi, rsi, rdx, rax, rcx; + __m256i y4; + __m512i z0, z1, z2, z3; + + rdi = (word64)(size_t)m; + rsi = (word64)(size_t)p; + + rdx = (word64)(0x680); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + rax = (word64)(0xfcc1); + z1 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z1 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z1)); + z2 = _mm512_sub_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 0))); + z3 = _mm512_srai_epi16(z2, 15); + z2 = _mm512_xor_si512(z2, z3); + z2 = _mm512_add_epi16(z2, z1); + y4 = _mm512_extracti64x4_epi64(z2, 1); + z2 = _mm512_zextsi256_si512(_mm256_packs_epi16(_mm512_castsi512_si256(z2), + y4)); + z2 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm512_castsi512_si256( + z2), 0xd8)); + rcx = (word32)((word32)_mm256_movemask_epi8(_mm512_castsi512_si256(z2))); + WC_S32(rdi, 0) = (word32)((word32)rcx); + z2 = _mm512_sub_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 64))); + z3 = _mm512_srai_epi16(z2, 15); + z2 = _mm512_xor_si512(z2, z3); + z2 = _mm512_add_epi16(z2, z1); + y4 = _mm512_extracti64x4_epi64(z2, 1); + z2 = _mm512_zextsi256_si512(_mm256_packs_epi16(_mm512_castsi512_si256(z2), + y4)); + z2 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm512_castsi512_si256( + z2), 0xd8)); + rcx = (word32)((word32)_mm256_movemask_epi8(_mm512_castsi512_si256(z2))); + WC_S32(rdi, 4) = (word32)((word32)rcx); + z2 = _mm512_sub_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 128))); + z3 = _mm512_srai_epi16(z2, 15); + z2 = _mm512_xor_si512(z2, z3); + z2 = _mm512_add_epi16(z2, z1); + y4 = _mm512_extracti64x4_epi64(z2, 1); + z2 = _mm512_zextsi256_si512(_mm256_packs_epi16(_mm512_castsi512_si256(z2), + y4)); + z2 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm512_castsi512_si256( + z2), 0xd8)); + rcx = (word32)((word32)_mm256_movemask_epi8(_mm512_castsi512_si256(z2))); + WC_S32(rdi, 8) = (word32)((word32)rcx); + z2 = _mm512_sub_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 192))); + z3 = _mm512_srai_epi16(z2, 15); + z2 = _mm512_xor_si512(z2, z3); + z2 = _mm512_add_epi16(z2, z1); + y4 = _mm512_extracti64x4_epi64(z2, 1); + z2 = _mm512_zextsi256_si512(_mm256_packs_epi16(_mm512_castsi512_si256(z2), + y4)); + z2 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm512_castsi512_si256( + z2), 0xd8)); + rcx = (word32)((word32)_mm256_movemask_epi8(_mm512_castsi512_si256(z2))); + WC_S32(rdi, 12) = (word32)((word32)rcx); + z2 = _mm512_sub_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 256))); + z3 = _mm512_srai_epi16(z2, 15); + z2 = _mm512_xor_si512(z2, z3); + z2 = _mm512_add_epi16(z2, z1); + y4 = _mm512_extracti64x4_epi64(z2, 1); + z2 = _mm512_zextsi256_si512(_mm256_packs_epi16(_mm512_castsi512_si256(z2), + y4)); + z2 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm512_castsi512_si256( + z2), 0xd8)); + rcx = (word32)((word32)_mm256_movemask_epi8(_mm512_castsi512_si256(z2))); + WC_S32(rdi, 16) = (word32)((word32)rcx); + z2 = _mm512_sub_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 320))); + z3 = _mm512_srai_epi16(z2, 15); + z2 = _mm512_xor_si512(z2, z3); + z2 = _mm512_add_epi16(z2, z1); + y4 = _mm512_extracti64x4_epi64(z2, 1); + z2 = _mm512_zextsi256_si512(_mm256_packs_epi16(_mm512_castsi512_si256(z2), + y4)); + z2 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm512_castsi512_si256( + z2), 0xd8)); + rcx = (word32)((word32)_mm256_movemask_epi8(_mm512_castsi512_si256(z2))); + WC_S32(rdi, 20) = (word32)((word32)rcx); + z2 = _mm512_sub_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 384))); + z3 = _mm512_srai_epi16(z2, 15); + z2 = _mm512_xor_si512(z2, z3); + z2 = _mm512_add_epi16(z2, z1); + y4 = _mm512_extracti64x4_epi64(z2, 1); + z2 = _mm512_zextsi256_si512(_mm256_packs_epi16(_mm512_castsi512_si256(z2), + y4)); + z2 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm512_castsi512_si256( + z2), 0xd8)); + rcx = (word32)((word32)_mm256_movemask_epi8(_mm512_castsi512_si256(z2))); + WC_S32(rdi, 24) = (word32)((word32)rcx); + z2 = _mm512_sub_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 448))); + z3 = _mm512_srai_epi16(z2, 15); + z2 = _mm512_xor_si512(z2, z3); + z2 = _mm512_add_epi16(z2, z1); + y4 = _mm512_extracti64x4_epi64(z2, 1); + z2 = _mm512_zextsi256_si512(_mm256_packs_epi16(_mm512_castsi512_si256(z2), + y4)); + z2 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm512_castsi512_si256( + z2), 0xd8)); + rcx = (word32)((word32)_mm256_movemask_epi8(_mm512_castsi512_si256(z2))); + WC_S32(rdi, 28) = (word32)((word32)rcx); +} + +XALIGNED(16) static const byte L_mlkem_compress_4_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x04, 0x08, 0x0c, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x00, 0x04, 0x08, 0x0c, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x00, 0x04, 0x08, 0x0c, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x00, 0x04, 0x08, 0x0c, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, +}; + +XALIGNED(16) static const word32 L_mlkem_compress_4_avx512_perm[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000004, 0x00000008, 0x0000000c, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_compress_4_avx512(byte* b, const sword16* p) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6; + + rdi = (word64)(size_t)b; + rsi = (word64)(size_t)p; + + z0 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_compress_4_avx512_shuf, + 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_compress_4_avx512_perm, + 0)); + rdx = (word64)(0x4ebf); + z2 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z2 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z2)); + rdx = (word64)(0x200); + z3 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z3 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z3)); + rdx = (word64)(0xf); + z4 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z4 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z4)); + z5 = _mm512_mulhi_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(rsi, 0))); + z5 = _mm512_mulhrs_epi16(z5, z3); + z5 = _mm512_and_si512(z5, z4); + z6 = _mm512_srli_epi32(z5, 12); + z5 = _mm512_or_si512(z5, z6); + z5 = _mm512_shuffle_epi8(z5, z0); + z5 = _mm512_permutexvar_epi32(z1, z5); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm512_castsi512_si128(z5)); + z5 = _mm512_mulhi_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(rsi, + 64))); + z5 = _mm512_mulhrs_epi16(z5, z3); + z5 = _mm512_and_si512(z5, z4); + z6 = _mm512_srli_epi32(z5, 12); + z5 = _mm512_or_si512(z5, z6); + z5 = _mm512_shuffle_epi8(z5, z0); + z5 = _mm512_permutexvar_epi32(z1, z5); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), _mm512_castsi512_si128(z5)); + z5 = _mm512_mulhi_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(rsi, + 128))); + z5 = _mm512_mulhrs_epi16(z5, z3); + z5 = _mm512_and_si512(z5, z4); + z6 = _mm512_srli_epi32(z5, 12); + z5 = _mm512_or_si512(z5, z6); + z5 = _mm512_shuffle_epi8(z5, z0); + z5 = _mm512_permutexvar_epi32(z1, z5); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), _mm512_castsi512_si128(z5)); + z5 = _mm512_mulhi_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(rsi, + 192))); + z5 = _mm512_mulhrs_epi16(z5, z3); + z5 = _mm512_and_si512(z5, z4); + z6 = _mm512_srli_epi32(z5, 12); + z5 = _mm512_or_si512(z5, z6); + z5 = _mm512_shuffle_epi8(z5, z0); + z5 = _mm512_permutexvar_epi32(z1, z5); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 48), _mm512_castsi512_si128(z5)); + z5 = _mm512_mulhi_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(rsi, + 256))); + z5 = _mm512_mulhrs_epi16(z5, z3); + z5 = _mm512_and_si512(z5, z4); + z6 = _mm512_srli_epi32(z5, 12); + z5 = _mm512_or_si512(z5, z6); + z5 = _mm512_shuffle_epi8(z5, z0); + z5 = _mm512_permutexvar_epi32(z1, z5); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 64), _mm512_castsi512_si128(z5)); + z5 = _mm512_mulhi_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(rsi, + 320))); + z5 = _mm512_mulhrs_epi16(z5, z3); + z5 = _mm512_and_si512(z5, z4); + z6 = _mm512_srli_epi32(z5, 12); + z5 = _mm512_or_si512(z5, z6); + z5 = _mm512_shuffle_epi8(z5, z0); + z5 = _mm512_permutexvar_epi32(z1, z5); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), _mm512_castsi512_si128(z5)); + z5 = _mm512_mulhi_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(rsi, + 384))); + z5 = _mm512_mulhrs_epi16(z5, z3); + z5 = _mm512_and_si512(z5, z4); + z6 = _mm512_srli_epi32(z5, 12); + z5 = _mm512_or_si512(z5, z6); + z5 = _mm512_shuffle_epi8(z5, z0); + z5 = _mm512_permutexvar_epi32(z1, z5); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 96), _mm512_castsi512_si128(z5)); + z5 = _mm512_mulhi_epi16(z2, _mm512_loadu_si512((const void*)WC_PR(rsi, + 448))); + z5 = _mm512_mulhrs_epi16(z5, z3); + z5 = _mm512_and_si512(z5, z4); + z6 = _mm512_srli_epi32(z5, 12); + z5 = _mm512_or_si512(z5, z6); + z5 = _mm512_shuffle_epi8(z5, z0); + z5 = _mm512_permutexvar_epi32(z1, z5); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 112), _mm512_castsi512_si128(z5)); +} + +XALIGNED(16) static const byte L_mlkem_decompress_4_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, + 0x04, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x05, + 0x06, 0x06, 0x06, 0x06, 0x07, 0x07, 0x07, 0x07, + 0x08, 0x08, 0x08, 0x08, 0x09, 0x09, 0x09, 0x09, + 0x0a, 0x0a, 0x0a, 0x0a, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0d, 0x0d, 0x0d, 0x0d, + 0x0e, 0x0e, 0x0e, 0x0e, 0x0f, 0x0f, 0x0f, 0x0f, +}; + +XALIGNED(16) static const word32 L_mlkem_decompress_4_avx512_mask[] + WC_X64I_UNUSED = { + 0x00f0000f, 0x00f0000f, 0x00f0000f, 0x00f0000f, + 0x00f0000f, 0x00f0000f, 0x00f0000f, 0x00f0000f, + 0x00f0000f, 0x00f0000f, 0x00f0000f, 0x00f0000f, + 0x00f0000f, 0x00f0000f, 0x00f0000f, 0x00f0000f, +}; + +XALIGNED(16) static const word32 L_mlkem_decompress_4_avx512_shift[] + WC_X64I_UNUSED = { + 0x00800800, 0x00800800, 0x00800800, 0x00800800, + 0x00800800, 0x00800800, 0x00800800, 0x00800800, + 0x00800800, 0x00800800, 0x00800800, 0x00800800, + 0x00800800, 0x00800800, 0x00800800, 0x00800800, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_decompress_4_avx512(sword16* p, const byte* b) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)b; + + z0 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_decompress_4_avx512_shuf, + 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_decompress_4_avx512_mask, + 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_4_avx512_shift, 0)); + rdx = (word64)(0xd01); + z3 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z3 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z3)); + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsi, 0))); + z4 = _mm512_shuffle_epi8(z4, z0); + z4 = _mm512_and_si512(z4, z1); + z4 = _mm512_mullo_epi16(z4, z2); + z4 = _mm512_mulhrs_epi16(z4, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z4); + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 16))); + z4 = _mm512_shuffle_epi8(z4, z0); + z4 = _mm512_and_si512(z4, z1); + z4 = _mm512_mullo_epi16(z4, z2); + z4 = _mm512_mulhrs_epi16(z4, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z4); + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 32))); + z4 = _mm512_shuffle_epi8(z4, z0); + z4 = _mm512_and_si512(z4, z1); + z4 = _mm512_mullo_epi16(z4, z2); + z4 = _mm512_mulhrs_epi16(z4, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z4); + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 48))); + z4 = _mm512_shuffle_epi8(z4, z0); + z4 = _mm512_and_si512(z4, z1); + z4 = _mm512_mullo_epi16(z4, z2); + z4 = _mm512_mulhrs_epi16(z4, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z4); + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 64))); + z4 = _mm512_shuffle_epi8(z4, z0); + z4 = _mm512_and_si512(z4, z1); + z4 = _mm512_mullo_epi16(z4, z2); + z4 = _mm512_mulhrs_epi16(z4, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 80))); + z4 = _mm512_shuffle_epi8(z4, z0); + z4 = _mm512_and_si512(z4, z1); + z4 = _mm512_mullo_epi16(z4, z2); + z4 = _mm512_mulhrs_epi16(z4, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z4); + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 96))); + z4 = _mm512_shuffle_epi8(z4, z0); + z4 = _mm512_and_si512(z4, z1); + z4 = _mm512_mullo_epi16(z4, z2); + z4 = _mm512_mulhrs_epi16(z4, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z4); + z4 = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 112))); + z4 = _mm512_shuffle_epi8(z4, z0); + z4 = _mm512_and_si512(z4, z1); + z4 = _mm512_mullo_epi16(z4, z2); + z4 = _mm512_mulhrs_epi16(z4, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z4); +} + +XALIGNED(16) static const word16 L_mlkem_decompress_5_avx512_q[] + WC_X64I_UNUSED = { + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_5_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, + 0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, + 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x06, 0x07, + 0x07, 0x08, 0x08, 0x08, 0x08, 0x09, 0x09, 0x09, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, + 0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, + 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x06, 0x07, + 0x07, 0x08, 0x08, 0x08, 0x08, 0x09, 0x09, 0x09, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_5_avx512_mask[] + WC_X64I_UNUSED = { + 0x001f, 0x03e0, 0x007c, 0x0f80, 0x01f0, 0x003e, 0x07c0, 0x00f8, + 0x001f, 0x03e0, 0x007c, 0x0f80, 0x01f0, 0x003e, 0x07c0, 0x00f8, + 0x001f, 0x03e0, 0x007c, 0x0f80, 0x01f0, 0x003e, 0x07c0, 0x00f8, + 0x001f, 0x03e0, 0x007c, 0x0f80, 0x01f0, 0x003e, 0x07c0, 0x00f8, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_5_avx512_shift[] + WC_X64I_UNUSED = { + 0x0400, 0x0020, 0x0100, 0x0008, 0x0040, 0x0200, 0x0010, 0x0080, + 0x0400, 0x0020, 0x0100, 0x0008, 0x0040, 0x0200, 0x0010, 0x0080, + 0x0400, 0x0020, 0x0100, 0x0008, 0x0040, 0x0200, 0x0010, 0x0080, + 0x0400, 0x0020, 0x0100, 0x0008, 0x0040, 0x0200, 0x0010, 0x0080, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_decompress_5_avx512(sword16* p, const byte* b) +{ + word64 rdi, rsi, rdx; + __m256i y5; + __m512i z0, z1, z2, z3, z4; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)b; + + z0 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_decompress_5_avx512_q, + 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_decompress_5_avx512_shuf, + 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_decompress_5_avx512_mask, + 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_5_avx512_shift, 0)); + z4 = _mm512_zextsi256_si512(_mm256_broadcastsi128_si256(_mm_loadu_si128(( + const __m128i*)WC_PR(rsi, 0)))); + y5 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 10))); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z1); + z4 = _mm512_and_si512(z4, z2); + z4 = _mm512_mullo_epi16(z4, z3); + z4 = _mm512_mulhrs_epi16(z4, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z4); + z4 = _mm512_zextsi256_si512(_mm256_broadcastsi128_si256(_mm_loadu_si128(( + const __m128i*)WC_PR(rsi, 20)))); + y5 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 30))); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z1); + z4 = _mm512_and_si512(z4, z2); + z4 = _mm512_mullo_epi16(z4, z3); + z4 = _mm512_mulhrs_epi16(z4, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z4); + z4 = _mm512_zextsi256_si512(_mm256_broadcastsi128_si256(_mm_loadu_si128(( + const __m128i*)WC_PR(rsi, 40)))); + y5 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 50))); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z1); + z4 = _mm512_and_si512(z4, z2); + z4 = _mm512_mullo_epi16(z4, z3); + z4 = _mm512_mulhrs_epi16(z4, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z4); + z4 = _mm512_zextsi256_si512(_mm256_broadcastsi128_si256(_mm_loadu_si128(( + const __m128i*)WC_PR(rsi, 60)))); + y5 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 70))); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z1); + z4 = _mm512_and_si512(z4, z2); + z4 = _mm512_mullo_epi16(z4, z3); + z4 = _mm512_mulhrs_epi16(z4, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z4); + z4 = _mm512_zextsi256_si512(_mm256_broadcastsi128_si256(_mm_loadu_si128(( + const __m128i*)WC_PR(rsi, 80)))); + y5 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 90))); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z1); + z4 = _mm512_and_si512(z4, z2); + z4 = _mm512_mullo_epi16(z4, z3); + z4 = _mm512_mulhrs_epi16(z4, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + z4 = _mm512_zextsi256_si512(_mm256_broadcastsi128_si256(_mm_loadu_si128(( + const __m128i*)WC_PR(rsi, 100)))); + y5 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 110))); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z1); + z4 = _mm512_and_si512(z4, z2); + z4 = _mm512_mullo_epi16(z4, z3); + z4 = _mm512_mulhrs_epi16(z4, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z4); + z4 = _mm512_zextsi256_si512(_mm256_broadcastsi128_si256(_mm_loadu_si128(( + const __m128i*)WC_PR(rsi, 120)))); + y5 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)WC_PR(rsi, + 130))); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z1); + z4 = _mm512_and_si512(z4, z2); + z4 = _mm512_mullo_epi16(z4, z3); + z4 = _mm512_mulhrs_epi16(z4, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z4); + z4 = _mm512_zextsi256_si512(_mm256_broadcastsi128_si256(_mm_loadu_si128(( + const __m128i*)WC_PR(rsi, 140)))); + y5 = _mm256_zextsi128_si256(_mm_loadl_epi64((const __m128i*)WC_PR(rsi, + 150))); + rdx = (word64)((word64)WC_L16(rsi, 158)); + y5 = _mm256_zextsi128_si256(_mm_insert_epi64(_mm256_castsi256_si128(y5), ( + long long)rdx, 1)); + y5 = _mm256_inserti128_si256(y5, _mm256_castsi256_si128(y5), 1); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z1); + z4 = _mm512_and_si512(z4, z2); + z4 = _mm512_mullo_epi16(z4, z3); + z4 = _mm512_mulhrs_epi16(z4, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z4); +} + +XALIGNED(16) static const word32 L_mlkem_decompress_10_avx512_mask[] + WC_X64I_UNUSED = { + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, +}; + +XALIGNED(32) static const word64 L_mlkem_decompress_10_avx512_sllv[] + WC_X64I_UNUSED = { + 0x0000000000000004ULL, 0x0000000000000004ULL, + 0x0000000000000004ULL, 0x0000000000000004ULL, + 0x0000000000000004ULL, 0x0000000000000004ULL, + 0x0000000000000004ULL, 0x0000000000000004ULL, +}; + +XALIGNED(16) static const word32 L_mlkem_decompress_10_avx512_q[] + WC_X64I_UNUSED = { + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_10_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, + 0x05, 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, + 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, + 0x07, 0x08, 0x08, 0x09, 0x09, 0x0a, 0x0a, 0x0b, + 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, + 0x05, 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, + 0x02, 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, + 0x07, 0x08, 0x08, 0x09, 0x09, 0x0a, 0x0a, 0x0b, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_decompress_10_avx512(sword16* p, const byte* r, int n) +{ + word64 rdi, rsi, rdx; + __m256i y5 = _mm256_setzero_si256(); + __m512i z0, z1, z2, z3, z4 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + rdx = (word64)(word32)n; + + z0 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_10_avx512_mask, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_decompress_10_avx512_q, + 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_10_avx512_shuf, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_10_avx512_sllv, 0)); +L_mlkem_decompress_10_avx512_start: + z4 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 0)), 0x94)); + y5 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 20)), 0x94); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z2); + z4 = _mm512_sllv_epi32(z4, z3); + z4 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z4 = _mm512_mulhrs_epi16(z4, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z4); + z4 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 40)), 0x94)); + y5 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 60)), 0x94); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z2); + z4 = _mm512_sllv_epi32(z4, z3); + z4 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z4 = _mm512_mulhrs_epi16(z4, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z4); + z4 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 80)), 0x94)); + y5 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 100)), 0x94); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z2); + z4 = _mm512_sllv_epi32(z4, z3); + z4 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z4 = _mm512_mulhrs_epi16(z4, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z4); + z4 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 120)), 0x94)); + y5 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 140)), 0x94); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z2); + z4 = _mm512_sllv_epi32(z4, z3); + z4 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z4 = _mm512_mulhrs_epi16(z4, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z4); + z4 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 160)), 0x94)); + y5 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 180)), 0x94); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z2); + z4 = _mm512_sllv_epi32(z4, z3); + z4 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z4 = _mm512_mulhrs_epi16(z4, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + z4 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 200)), 0x94)); + y5 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 220)), 0x94); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z2); + z4 = _mm512_sllv_epi32(z4, z3); + z4 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z4 = _mm512_mulhrs_epi16(z4, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z4); + z4 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 240)), 0x94)); + y5 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 260)), 0x94); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z2); + z4 = _mm512_sllv_epi32(z4, z3); + z4 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z4 = _mm512_mulhrs_epi16(z4, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z4); + z4 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 280)), 0x94)); + y5 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 300)), 0x94); + z4 = _mm512_inserti64x4(z4, y5, 1); + z4 = _mm512_shuffle_epi8(z4, z2); + z4 = _mm512_sllv_epi32(z4, z3); + z4 = _mm512_srli_epi16(z4, 1); + z4 = _mm512_and_si512(z4, z0); + z4 = _mm512_mulhrs_epi16(z4, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z4); + rsi = (word64)(rsi + 0x140); + rdi = (word64)(rdi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_decompress_10_avx512_start; + } +} + +XALIGNED(16) static const word16 L_mlkem_decompress_11_avx512_q[] + WC_X64I_UNUSED = { + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_11_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x04, 0x05, + 0x05, 0x06, 0x06, 0x07, 0x08, 0x09, 0x09, 0x0a, + 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x07, 0x08, + 0x08, 0x09, 0x09, 0x0a, 0x0b, 0x0c, 0x0c, 0x0d, + 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x04, 0x05, + 0x05, 0x06, 0x06, 0x07, 0x08, 0x09, 0x09, 0x0a, + 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x07, 0x08, + 0x08, 0x09, 0x09, 0x0a, 0x0b, 0x0c, 0x0c, 0x0d, +}; + +XALIGNED(16) static const word32 L_mlkem_decompress_11_avx512_srlvd[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000000, 0x00000000, + 0x00000000, 0x00000001, 0x00000000, 0x00000000, + 0x00000000, 0x00000001, 0x00000000, 0x00000000, + 0x00000000, 0x00000001, 0x00000000, 0x00000000, +}; + +XALIGNED(32) static const word64 L_mlkem_decompress_11_avx512_srlvq[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_11_avx512_shift[] + WC_X64I_UNUSED = { + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_11_avx512_mask[] + WC_X64I_UNUSED = { + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_decompress_11_avx512(sword16* p, const byte* r, int n) +{ + word64 rdi, rsi, rdx; + __m256i y7 = _mm256_setzero_si256(); + __m512i z0, z1, z2, z3, z4, z5, z6 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + rdx = (word64)(word32)n; + + z0 = _mm512_loadu_si512((const void*)WC_PR(L_mlkem_decompress_11_avx512_q, + 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_shuf, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_srlvd, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_srlvq, 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_shift, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_mask, 0)); +L_mlkem_decompress_11_avx512_start: + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 0)), 0x94)); + y7 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 22)), 0x94); + z6 = _mm512_inserti64x4(z6, y7, 1); + z6 = _mm512_shuffle_epi8(z6, z1); + z6 = _mm512_srlv_epi32(z6, z2); + z6 = _mm512_srlv_epi64(z6, z3); + z6 = _mm512_mullo_epi16(z6, z4); + z6 = _mm512_srli_epi16(z6, 1); + z6 = _mm512_and_si512(z6, z5); + z6 = _mm512_mulhrs_epi16(z6, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z6); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 44)), 0x94)); + y7 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 66)), 0x94); + z6 = _mm512_inserti64x4(z6, y7, 1); + z6 = _mm512_shuffle_epi8(z6, z1); + z6 = _mm512_srlv_epi32(z6, z2); + z6 = _mm512_srlv_epi64(z6, z3); + z6 = _mm512_mullo_epi16(z6, z4); + z6 = _mm512_srli_epi16(z6, 1); + z6 = _mm512_and_si512(z6, z5); + z6 = _mm512_mulhrs_epi16(z6, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z6); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 88)), 0x94)); + y7 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 110)), 0x94); + z6 = _mm512_inserti64x4(z6, y7, 1); + z6 = _mm512_shuffle_epi8(z6, z1); + z6 = _mm512_srlv_epi32(z6, z2); + z6 = _mm512_srlv_epi64(z6, z3); + z6 = _mm512_mullo_epi16(z6, z4); + z6 = _mm512_srli_epi16(z6, 1); + z6 = _mm512_and_si512(z6, z5); + z6 = _mm512_mulhrs_epi16(z6, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z6); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 132)), 0x94)); + y7 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 154)), 0x94); + z6 = _mm512_inserti64x4(z6, y7, 1); + z6 = _mm512_shuffle_epi8(z6, z1); + z6 = _mm512_srlv_epi32(z6, z2); + z6 = _mm512_srlv_epi64(z6, z3); + z6 = _mm512_mullo_epi16(z6, z4); + z6 = _mm512_srli_epi16(z6, 1); + z6 = _mm512_and_si512(z6, z5); + z6 = _mm512_mulhrs_epi16(z6, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z6); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 176)), 0x94)); + y7 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 198)), 0x94); + z6 = _mm512_inserti64x4(z6, y7, 1); + z6 = _mm512_shuffle_epi8(z6, z1); + z6 = _mm512_srlv_epi32(z6, z2); + z6 = _mm512_srlv_epi64(z6, z3); + z6 = _mm512_mullo_epi16(z6, z4); + z6 = _mm512_srli_epi16(z6, 1); + z6 = _mm512_and_si512(z6, z5); + z6 = _mm512_mulhrs_epi16(z6, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z6); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 220)), 0x94)); + y7 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 242)), 0x94); + z6 = _mm512_inserti64x4(z6, y7, 1); + z6 = _mm512_shuffle_epi8(z6, z1); + z6 = _mm512_srlv_epi32(z6, z2); + z6 = _mm512_srlv_epi64(z6, z3); + z6 = _mm512_mullo_epi16(z6, z4); + z6 = _mm512_srli_epi16(z6, 1); + z6 = _mm512_and_si512(z6, z5); + z6 = _mm512_mulhrs_epi16(z6, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z6); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 264)), 0x94)); + y7 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 286)), 0x94); + z6 = _mm512_inserti64x4(z6, y7, 1); + z6 = _mm512_shuffle_epi8(z6, z1); + z6 = _mm512_srlv_epi32(z6, z2); + z6 = _mm512_srlv_epi64(z6, z3); + z6 = _mm512_mullo_epi16(z6, z4); + z6 = _mm512_srli_epi16(z6, 1); + z6 = _mm512_and_si512(z6, z5); + z6 = _mm512_mulhrs_epi16(z6, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z6); + z6 = _mm512_zextsi256_si512(_mm256_permute4x64_epi64(_mm256_loadu_si256(( + const __m256i*)WC_PR(rsi, 308)), 0x94)); + y7 = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)WC_PR(rsi, + 330)), 0x94); + z6 = _mm512_inserti64x4(z6, y7, 1); + z6 = _mm512_shuffle_epi8(z6, z1); + z6 = _mm512_srlv_epi32(z6, z2); + z6 = _mm512_srlv_epi64(z6, z3); + z6 = _mm512_mullo_epi16(z6, z4); + z6 = _mm512_srli_epi16(z6, 1); + z6 = _mm512_and_si512(z6, z5); + z6 = _mm512_mulhrs_epi16(z6, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z6); + rsi = (word64)(rsi + 0x160); + rdi = (word64)(rdi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_decompress_11_avx512_start; + } +} + +XALIGNED(16) static const word16 L_mlkem_compress_5_avx512_shift1[] + WC_X64I_UNUSED = { + 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, + 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, +}; + +XALIGNED(16) static const word32 L_mlkem_compress_5_avx512_shift2[] + WC_X64I_UNUSED = { + 0x04000001, 0x04000001, 0x04000001, 0x04000001, + 0x04000001, 0x04000001, 0x04000001, 0x04000001, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_5_avx512_shlv[] + WC_X64I_UNUSED = { + 0x000000000000000cULL, 0x000000000000000cULL, + 0x000000000000000cULL, 0x000000000000000cULL, +}; + +XALIGNED(16) static const byte L_mlkem_compress_5_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0xff, + 0x09, 0x0a, 0x0b, 0x0c, 0xff, 0x00, 0x01, 0x02, + 0x03, 0x04, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_compress_5_avx512(byte* b, const sword16* p) +{ + word64 rdi, rsi, rdx; + __m256i y3, y4, y5, y6, y8, y9; + __m512i z0, z1, z2, z7; + + rdi = (word64)(size_t)b; + rsi = (word64)(size_t)p; + + rdx = (word64)(0x4ebf); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + rdx = (word64)(0x400); + z1 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z1 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z1)); + rdx = (word64)(0x1f); + z2 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z2 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z2)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx512_shift1, 0)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx512_shift2, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx512_shlv, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx512_shuf, 0)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 0))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y9 = _mm512_extracti64x4_epi64(z7, 1); + y8 = _mm256_packus_epi16(_mm512_castsi512_si256(z7), y9); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srlv_epi64(y8, y5); + y8 = _mm256_shuffle_epi8(y8, y6); + y9 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y8, 1)); + y8 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9), _mm256_castsi256_si128(y6))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y8)); + WC_S32(rdi, 16) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y9)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 64))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y9 = _mm512_extracti64x4_epi64(z7, 1); + y8 = _mm256_packus_epi16(_mm512_castsi512_si256(z7), y9); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srlv_epi64(y8, y5); + y8 = _mm256_shuffle_epi8(y8, y6); + y9 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y8, 1)); + y8 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9), _mm256_castsi256_si128(y6))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 20), _mm256_castsi256_si128(y8)); + WC_S32(rdi, 36) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y9)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 128))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y9 = _mm512_extracti64x4_epi64(z7, 1); + y8 = _mm256_packus_epi16(_mm512_castsi512_si256(z7), y9); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srlv_epi64(y8, y5); + y8 = _mm256_shuffle_epi8(y8, y6); + y9 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y8, 1)); + y8 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9), _mm256_castsi256_si128(y6))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 40), _mm256_castsi256_si128(y8)); + WC_S32(rdi, 56) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y9)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 192))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y9 = _mm512_extracti64x4_epi64(z7, 1); + y8 = _mm256_packus_epi16(_mm512_castsi512_si256(z7), y9); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srlv_epi64(y8, y5); + y8 = _mm256_shuffle_epi8(y8, y6); + y9 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y8, 1)); + y8 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9), _mm256_castsi256_si128(y6))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 60), _mm256_castsi256_si128(y8)); + WC_S32(rdi, 76) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y9)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 256))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y9 = _mm512_extracti64x4_epi64(z7, 1); + y8 = _mm256_packus_epi16(_mm512_castsi512_si256(z7), y9); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srlv_epi64(y8, y5); + y8 = _mm256_shuffle_epi8(y8, y6); + y9 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y8, 1)); + y8 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9), _mm256_castsi256_si128(y6))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), _mm256_castsi256_si128(y8)); + WC_S32(rdi, 96) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y9)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 320))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y9 = _mm512_extracti64x4_epi64(z7, 1); + y8 = _mm256_packus_epi16(_mm512_castsi512_si256(z7), y9); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srlv_epi64(y8, y5); + y8 = _mm256_shuffle_epi8(y8, y6); + y9 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y8, 1)); + y8 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9), _mm256_castsi256_si128(y6))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 100), _mm256_castsi256_si128(y8)); + WC_S32(rdi, 116) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y9)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 384))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y9 = _mm512_extracti64x4_epi64(z7, 1); + y8 = _mm256_packus_epi16(_mm512_castsi512_si256(z7), y9); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srlv_epi64(y8, y5); + y8 = _mm256_shuffle_epi8(y8, y6); + y9 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y8, 1)); + y8 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9), _mm256_castsi256_si128(y6))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 120), _mm256_castsi256_si128(y8)); + WC_S32(rdi, 136) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y9)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 448))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y9 = _mm512_extracti64x4_epi64(z7, 1); + y8 = _mm256_packus_epi16(_mm512_castsi512_si256(z7), y9); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srlv_epi64(y8, y5); + y8 = _mm256_shuffle_epi8(y8, y6); + y9 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y8, 1)); + y8 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y8), + _mm256_castsi256_si128(y9), _mm256_castsi256_si128(y6))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 140), _mm256_castsi256_si128(y8)); + WC_S32(rdi, 156) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y9)); +} + +XALIGNED(32) static const word64 L_mlkem_compress_10_avx512_shift[] + WC_X64I_UNUSED = { + 0x0400000104000001ULL, 0x0400000104000001ULL, + 0x0400000104000001ULL, 0x0400000104000001ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_10_avx512_shlv[] + WC_X64I_UNUSED = { + 0x000000000000000cULL, 0x000000000000000cULL, + 0x000000000000000cULL, 0x000000000000000cULL, +}; + +XALIGNED(16) static const byte L_mlkem_compress_10_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x09, 0x0a, 0x0b, 0x0c, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_compress_10_avx512(byte* r, const sword16* p, int n) +{ + word64 rdi, rsi, rdx, rax; + __m128i x12 = _mm_setzero_si128(); + __m256i y5, y6, y7, y11 = _mm256_setzero_si256(); + __m512i z0, z1, z2, z3, z4, z8 = _mm512_setzero_si512(), + z9 = _mm512_setzero_si512(), z10 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + rdx = (word64)(word32)n; + + rax = (word64)(0x4ebf); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + rax = (word64)(0xf); + z2 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z2 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z2)); + rax = (word64)(0x1000); + z3 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z3 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z3)); + rax = (word64)(0x3ff); + z4 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z4 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z4)); + z1 = _mm512_slli_epi16(z0, 3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_10_avx512_shift, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_10_avx512_shlv, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_10_avx512_shuf, 0)); +L_mlkem_compress_10_avx512_start: + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + y11 = _mm256_madd_epi16(_mm512_castsi512_si256(z8), y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 16) = (word32)_mm_cvtsi128_si32(x12); + y11 = _mm512_extracti64x4_epi64(z8, 1); + y11 = _mm256_madd_epi16(y11, y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 20), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 36) = (word32)_mm_cvtsi128_si32(x12); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + y11 = _mm256_madd_epi16(_mm512_castsi512_si256(z8), y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 40), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 56) = (word32)_mm_cvtsi128_si32(x12); + y11 = _mm512_extracti64x4_epi64(z8, 1); + y11 = _mm256_madd_epi16(y11, y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 60), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 76) = (word32)_mm_cvtsi128_si32(x12); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + y11 = _mm256_madd_epi16(_mm512_castsi512_si256(z8), y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 96) = (word32)_mm_cvtsi128_si32(x12); + y11 = _mm512_extracti64x4_epi64(z8, 1); + y11 = _mm256_madd_epi16(y11, y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 100), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 116) = (word32)_mm_cvtsi128_si32(x12); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + y11 = _mm256_madd_epi16(_mm512_castsi512_si256(z8), y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 120), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 136) = (word32)_mm_cvtsi128_si32(x12); + y11 = _mm512_extracti64x4_epi64(z8, 1); + y11 = _mm256_madd_epi16(y11, y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 140), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 156) = (word32)_mm_cvtsi128_si32(x12); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + y11 = _mm256_madd_epi16(_mm512_castsi512_si256(z8), y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 160), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 176) = (word32)_mm_cvtsi128_si32(x12); + y11 = _mm512_extracti64x4_epi64(z8, 1); + y11 = _mm256_madd_epi16(y11, y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 180), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 196) = (word32)_mm_cvtsi128_si32(x12); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + y11 = _mm256_madd_epi16(_mm512_castsi512_si256(z8), y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 200), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 216) = (word32)_mm_cvtsi128_si32(x12); + y11 = _mm512_extracti64x4_epi64(z8, 1); + y11 = _mm256_madd_epi16(y11, y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 220), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 236) = (word32)_mm_cvtsi128_si32(x12); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + y11 = _mm256_madd_epi16(_mm512_castsi512_si256(z8), y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 240), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 256) = (word32)_mm_cvtsi128_si32(x12); + y11 = _mm512_extracti64x4_epi64(z8, 1); + y11 = _mm256_madd_epi16(y11, y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 260), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 276) = (word32)_mm_cvtsi128_si32(x12); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + y11 = _mm256_madd_epi16(_mm512_castsi512_si256(z8), y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 280), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 296) = (word32)_mm_cvtsi128_si32(x12); + y11 = _mm512_extracti64x4_epi64(z8, 1); + y11 = _mm256_madd_epi16(y11, y5); + y11 = _mm256_sllv_epi32(y11, y6); + y11 = _mm256_srli_epi64(y11, 12); + y11 = _mm256_shuffle_epi8(y11, y7); + x12 = _mm256_extracti128_si256(y11, 1); + y11 = _mm256_zextsi128_si256(_mm_blend_epi16(_mm256_castsi256_si128(y11), + x12, 0xe0)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 300), _mm256_castsi256_si128(y11)); + WC_S32(rdi, 316) = (word32)_mm_cvtsi128_si32(x12); + rdi = (word64)(rdi + 0x140); + rsi = (word64)(rsi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_compress_10_avx512_start; + } +} + +XALIGNED(32) static const word64 L_mlkem_compress_11_avx512_shift[] + WC_X64I_UNUSED = { + 0x0800000108000001ULL, 0x0800000108000001ULL, + 0x0800000108000001ULL, 0x0800000108000001ULL, +}; + +XALIGNED(16) static const word32 L_mlkem_compress_11_avx512_shlvd[] + WC_X64I_UNUSED = { + 0x0000000a, 0x00000000, 0x0000000a, 0x00000000, + 0x0000000a, 0x00000000, 0x0000000a, 0x00000000, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_11_avx512_shrvq[] + WC_X64I_UNUSED = { + 0x000000000000000aULL, 0x000000000000001eULL, + 0x000000000000000aULL, 0x000000000000001eULL, +}; + +XALIGNED(16) static const byte L_mlkem_compress_11_avx512_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, +}; + +WC_X64I_TARGET("avx512f,avx512bw") +WOLFSSL_LOCAL void mlkem_compress_11_avx512(byte* r, const sword16* p, int n) +{ + word64 rdi, rsi, rdx, rax; + __m256i y5, y6, y7, y8, y12 = _mm256_setzero_si256(), + y13 = _mm256_setzero_si256(); + __m512i z0, z1, z2, z3, z4, z9 = _mm512_setzero_si512(), + z10 = _mm512_setzero_si512(), z11 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + rdx = (word64)(word32)n; + + rax = (word64)(0x4ebf); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + rax = (word64)(0x24); + z2 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z2 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z2)); + rax = (word64)(0x2000); + z3 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z3 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z3)); + rax = (word64)(0x7ff); + z4 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z4 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z4)); + z1 = _mm512_slli_epi16(z0, 3); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx512_shift, 0)); + y6 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx512_shlvd, 0)); + y7 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx512_shrvq, 0)); + y8 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_11_avx512_shuf, 0)); +L_mlkem_compress_11_avx512_start: + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z10 = _mm512_mullo_epi16(z9, z1); + z11 = _mm512_add_epi16(z9, z2); + z9 = _mm512_slli_epi16(z9, 3); + z9 = _mm512_mulhi_epi16(z9, z0); + z11 = _mm512_sub_epi16(z10, z11); + z10 = _mm512_andnot_si512(z10, z11); + z10 = _mm512_srli_epi16(z10, 15); + z9 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_mulhrs_epi16(z9, z3); + z9 = _mm512_and_si512(z9, z4); + y12 = _mm256_madd_epi16(_mm512_castsi512_si256(z9), y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 16), _mm256_castsi256_si128(y13)); + y12 = _mm512_extracti64x4_epi64(z9, 1); + y12 = _mm256_madd_epi16(y12, y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 22), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 38), _mm256_castsi256_si128(y13)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z10 = _mm512_mullo_epi16(z9, z1); + z11 = _mm512_add_epi16(z9, z2); + z9 = _mm512_slli_epi16(z9, 3); + z9 = _mm512_mulhi_epi16(z9, z0); + z11 = _mm512_sub_epi16(z10, z11); + z10 = _mm512_andnot_si512(z10, z11); + z10 = _mm512_srli_epi16(z10, 15); + z9 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_mulhrs_epi16(z9, z3); + z9 = _mm512_and_si512(z9, z4); + y12 = _mm256_madd_epi16(_mm512_castsi512_si256(z9), y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 44), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 60), _mm256_castsi256_si128(y13)); + y12 = _mm512_extracti64x4_epi64(z9, 1); + y12 = _mm256_madd_epi16(y12, y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 66), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 82), _mm256_castsi256_si128(y13)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z10 = _mm512_mullo_epi16(z9, z1); + z11 = _mm512_add_epi16(z9, z2); + z9 = _mm512_slli_epi16(z9, 3); + z9 = _mm512_mulhi_epi16(z9, z0); + z11 = _mm512_sub_epi16(z10, z11); + z10 = _mm512_andnot_si512(z10, z11); + z10 = _mm512_srli_epi16(z10, 15); + z9 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_mulhrs_epi16(z9, z3); + z9 = _mm512_and_si512(z9, z4); + y12 = _mm256_madd_epi16(_mm512_castsi512_si256(z9), y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 88), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 104), _mm256_castsi256_si128(y13)); + y12 = _mm512_extracti64x4_epi64(z9, 1); + y12 = _mm256_madd_epi16(y12, y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 110), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 126), _mm256_castsi256_si128(y13)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z10 = _mm512_mullo_epi16(z9, z1); + z11 = _mm512_add_epi16(z9, z2); + z9 = _mm512_slli_epi16(z9, 3); + z9 = _mm512_mulhi_epi16(z9, z0); + z11 = _mm512_sub_epi16(z10, z11); + z10 = _mm512_andnot_si512(z10, z11); + z10 = _mm512_srli_epi16(z10, 15); + z9 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_mulhrs_epi16(z9, z3); + z9 = _mm512_and_si512(z9, z4); + y12 = _mm256_madd_epi16(_mm512_castsi512_si256(z9), y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 132), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 148), _mm256_castsi256_si128(y13)); + y12 = _mm512_extracti64x4_epi64(z9, 1); + y12 = _mm256_madd_epi16(y12, y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 154), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 170), _mm256_castsi256_si128(y13)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z10 = _mm512_mullo_epi16(z9, z1); + z11 = _mm512_add_epi16(z9, z2); + z9 = _mm512_slli_epi16(z9, 3); + z9 = _mm512_mulhi_epi16(z9, z0); + z11 = _mm512_sub_epi16(z10, z11); + z10 = _mm512_andnot_si512(z10, z11); + z10 = _mm512_srli_epi16(z10, 15); + z9 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_mulhrs_epi16(z9, z3); + z9 = _mm512_and_si512(z9, z4); + y12 = _mm256_madd_epi16(_mm512_castsi512_si256(z9), y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 176), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 192), _mm256_castsi256_si128(y13)); + y12 = _mm512_extracti64x4_epi64(z9, 1); + y12 = _mm256_madd_epi16(y12, y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 198), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 214), _mm256_castsi256_si128(y13)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z10 = _mm512_mullo_epi16(z9, z1); + z11 = _mm512_add_epi16(z9, z2); + z9 = _mm512_slli_epi16(z9, 3); + z9 = _mm512_mulhi_epi16(z9, z0); + z11 = _mm512_sub_epi16(z10, z11); + z10 = _mm512_andnot_si512(z10, z11); + z10 = _mm512_srli_epi16(z10, 15); + z9 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_mulhrs_epi16(z9, z3); + z9 = _mm512_and_si512(z9, z4); + y12 = _mm256_madd_epi16(_mm512_castsi512_si256(z9), y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 220), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 236), _mm256_castsi256_si128(y13)); + y12 = _mm512_extracti64x4_epi64(z9, 1); + y12 = _mm256_madd_epi16(y12, y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 242), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 258), _mm256_castsi256_si128(y13)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z10 = _mm512_mullo_epi16(z9, z1); + z11 = _mm512_add_epi16(z9, z2); + z9 = _mm512_slli_epi16(z9, 3); + z9 = _mm512_mulhi_epi16(z9, z0); + z11 = _mm512_sub_epi16(z10, z11); + z10 = _mm512_andnot_si512(z10, z11); + z10 = _mm512_srli_epi16(z10, 15); + z9 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_mulhrs_epi16(z9, z3); + z9 = _mm512_and_si512(z9, z4); + y12 = _mm256_madd_epi16(_mm512_castsi512_si256(z9), y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 264), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 280), _mm256_castsi256_si128(y13)); + y12 = _mm512_extracti64x4_epi64(z9, 1); + y12 = _mm256_madd_epi16(y12, y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 286), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 302), _mm256_castsi256_si128(y13)); + z9 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z10 = _mm512_mullo_epi16(z9, z1); + z11 = _mm512_add_epi16(z9, z2); + z9 = _mm512_slli_epi16(z9, 3); + z9 = _mm512_mulhi_epi16(z9, z0); + z11 = _mm512_sub_epi16(z10, z11); + z10 = _mm512_andnot_si512(z10, z11); + z10 = _mm512_srli_epi16(z10, 15); + z9 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_mulhrs_epi16(z9, z3); + z9 = _mm512_and_si512(z9, z4); + y12 = _mm256_madd_epi16(_mm512_castsi512_si256(z9), y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 308), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 324), _mm256_castsi256_si128(y13)); + y12 = _mm512_extracti64x4_epi64(z9, 1); + y12 = _mm256_madd_epi16(y12, y5); + y12 = _mm256_sllv_epi32(y12, y6); + y13 = _mm256_bsrli_epi128(y12, 8); + y12 = _mm256_srlv_epi64(y12, y7); + y13 = _mm256_slli_epi64(y13, 34); + y12 = _mm256_add_epi64(y12, y13); + y12 = _mm256_shuffle_epi8(y12, y8); + y13 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y12, 1)); + y12 = _mm256_zextsi128_si256(_mm_blendv_epi8(_mm256_castsi256_si128(y12), + _mm256_castsi256_si128(y13), _mm256_castsi256_si128(y8))); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 330), _mm256_castsi256_si128(y12)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 346), _mm256_castsi256_si128(y13)); + rdi = (word64)(rdi + 0x160); + rsi = (word64)(rsi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_compress_11_avx512_start; + } +} + +#endif /* HAVE_INTEL_AVX512 */ +#ifdef HAVE_INTEL_AVX512_VBMI2 +XALIGNED(16) static const word32 L_mlkem_rej_uniform_avx512_vbmi2_perm[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000002, 0x00000000, + 0x00000003, 0x00000004, 0x00000005, 0x00000000, + 0x00000006, 0x00000007, 0x00000008, 0x00000000, + 0x00000009, 0x0000000a, 0x0000000b, 0x00000000, +}; + +XALIGNED(16) static const byte L_mlkem_rej_uniform_avx512_vbmi2_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_avx512_vbmi2_mask[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi2,bmi") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx512_vbmi2(sword16* p, + unsigned int len, const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, rbx = 0, r8 = 0, r9 = 0; + __m512i z0, z1, z2, z3, z4 = _mm512_setzero_si512(), + z5 = _mm512_setzero_si512(); + __mmask32 k1; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + rcx = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + r10 = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r10)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_avx512_vbmi2_perm, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_avx512_vbmi2_shuf, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_avx512_vbmi2_mask, 0)); +L_mlkem_rej_uniform_avx512_vbmi2_fast: + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_avx512_vbmi2_fast_end; + } + if ((sword32)((word32)rcx) < (sword32)(0x40)) { + goto L_mlkem_rej_uniform_avx512_vbmi2_fast_end; + } + z4 = _mm512_permutexvar_epi32(z1, _mm512_loadu_si512((const void*)WC_PR(rdx, + 0))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + k1 = _mm512_cmp_epi16_mask(z4, z0, 1); + _mm512_mask_compressstoreu_epi16(WC_PW(rdi, 0), k1, z4); + rbx = (word32)((word32)k1); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + rcx = (word32)((word32)rcx - 0x30); + goto L_mlkem_rej_uniform_avx512_vbmi2_fast; +L_mlkem_rej_uniform_avx512_vbmi2_fast_end: +L_mlkem_rej_uniform_avx512_vbmi2_tail: + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_vbmi2_tail_end; + } + if ((sword32)((word32)rcx) < (sword32)(3)) { + goto L_mlkem_rej_uniform_avx512_vbmi2_tail_end; + } + r8 = (word32)((word32)WC_L16(rdx, 0)); + r8 = (word32)((word32)r8 & 0xfff); + r9 = (word32)((word32)WC_L16(rdx, 1)); + r9 = (word32)((word32)r9 >> 4); + if ((sword32)((word32)r8) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_avx512_vbmi2_skip0; + } + WC_S16(rdi, 0) = (word16)((word16)r8); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_vbmi2_tail_end; + } +L_mlkem_rej_uniform_avx512_vbmi2_skip0: + if ((sword32)((word32)r9) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_avx512_vbmi2_skip1; + } + WC_S16(rdi, 0) = (word16)((word16)r9); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_vbmi2_tail_end; + } +L_mlkem_rej_uniform_avx512_vbmi2_skip1: + rdx = (word64)(rdx + 3); + rcx = (word32)((word32)rcx - 3); + goto L_mlkem_rej_uniform_avx512_vbmi2_tail; +L_mlkem_rej_uniform_avx512_vbmi2_tail_end: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; +} + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_n_avx512_vbmi2_perm[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000002, 0x00000000, + 0x00000003, 0x00000004, 0x00000005, 0x00000000, + 0x00000006, 0x00000007, 0x00000008, 0x00000000, + 0x00000009, 0x0000000a, 0x0000000b, 0x00000000, +}; + +XALIGNED(16) static const byte L_mlkem_rej_uniform_n_avx512_vbmi2_shuf[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_n_avx512_vbmi2_mask[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi2,bmi") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx512_vbmi2(sword16* p, + unsigned int len, const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, rbx = 0, r8 = 0, r9 = 0; + __m512i z0, z1, z2, z3, z4 = _mm512_setzero_si512(), + z5 = _mm512_setzero_si512(); + __mmask32 k1; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + rcx = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + r10 = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r10)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_vbmi2_perm, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_vbmi2_shuf, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_vbmi2_mask, 0)); +L_mlkem_rej_uniform_n_avx512_vbmi2_fast: + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi2_fast_end; + } + if ((sword32)((word32)rcx) < (sword32)(0x40)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi2_fast_end; + } + z4 = _mm512_permutexvar_epi32(z1, _mm512_loadu_si512((const void*)WC_PR(rdx, + 0))); + z4 = _mm512_shuffle_epi8(z4, z2); + z5 = _mm512_andnot_si512(z3, z4); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z3, z5, 0xea); + k1 = _mm512_cmp_epi16_mask(z4, z0, 1); + _mm512_mask_compressstoreu_epi16(WC_PW(rdi, 0), k1, z4); + rbx = (word32)((word32)k1); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + rcx = (word32)((word32)rcx - 0x30); + goto L_mlkem_rej_uniform_n_avx512_vbmi2_fast; +L_mlkem_rej_uniform_n_avx512_vbmi2_fast_end: +L_mlkem_rej_uniform_n_avx512_vbmi2_tail: + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi2_tail_end; + } + if ((sword32)((word32)rcx) < (sword32)(3)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi2_tail_end; + } + r8 = (word32)((word32)WC_L16(rdx, 0)); + r8 = (word32)((word32)r8 & 0xfff); + r9 = (word32)((word32)WC_L16(rdx, 1)); + r9 = (word32)((word32)r9 >> 4); + if ((sword32)((word32)r8) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi2_skip0; + } + WC_S16(rdi, 0) = (word16)((word16)r8); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi2_tail_end; + } +L_mlkem_rej_uniform_n_avx512_vbmi2_skip0: + if ((sword32)((word32)r9) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi2_skip1; + } + WC_S16(rdi, 0) = (word16)((word16)r9); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi2_tail_end; + } +L_mlkem_rej_uniform_n_avx512_vbmi2_skip1: + rdx = (word64)(rdx + 3); + rcx = (word32)((word32)rcx - 3); + goto L_mlkem_rej_uniform_n_avx512_vbmi2_tail; +L_mlkem_rej_uniform_n_avx512_vbmi2_tail_end: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; +} + +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* WOLFSSL_HAVE_MLKEM */ +#ifdef WOLFSSL_HAVE_MLKEM +#ifndef NO_AVX512_SUPPORT +#ifndef NO_AVX512_VBMI_SUPPORT +#ifndef HAVE_INTEL_AVX512_VBMI +#define HAVE_INTEL_AVX512_VBMI +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* NO_AVX512_VBMI_SUPPORT */ +#endif /* NO_AVX512_SUPPORT */ +#ifdef HAVE_INTEL_AVX512_VBMI +XALIGNED(16) static const byte L_mlkem_from_bytes_avx512_vbmi_lo[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x00, 0x03, 0x04, 0x05, 0x00, + 0x06, 0x07, 0x08, 0x00, 0x09, 0x0a, 0x0b, 0x00, + 0x0c, 0x0d, 0x0e, 0x00, 0x0f, 0x10, 0x11, 0x00, + 0x12, 0x13, 0x14, 0x00, 0x15, 0x16, 0x17, 0x00, + 0x18, 0x19, 0x1a, 0x00, 0x1b, 0x1c, 0x1d, 0x00, + 0x1e, 0x1f, 0x20, 0x00, 0x21, 0x22, 0x23, 0x00, + 0x24, 0x25, 0x26, 0x00, 0x27, 0x28, 0x29, 0x00, + 0x2a, 0x2b, 0x2c, 0x00, 0x2d, 0x2e, 0x2f, 0x00, +}; + +XALIGNED(16) static const byte L_mlkem_from_bytes_avx512_vbmi_hi[] + WC_X64I_UNUSED = { + 0x10, 0x11, 0x12, 0x00, 0x13, 0x14, 0x15, 0x00, + 0x16, 0x17, 0x18, 0x00, 0x19, 0x1a, 0x1b, 0x00, + 0x1c, 0x1d, 0x1e, 0x00, 0x1f, 0x20, 0x21, 0x00, + 0x22, 0x23, 0x24, 0x00, 0x25, 0x26, 0x27, 0x00, + 0x28, 0x29, 0x2a, 0x00, 0x2b, 0x2c, 0x2d, 0x00, + 0x2e, 0x2f, 0x30, 0x00, 0x31, 0x32, 0x33, 0x00, + 0x34, 0x35, 0x36, 0x00, 0x37, 0x38, 0x39, 0x00, + 0x3a, 0x3b, 0x3c, 0x00, 0x3d, 0x3e, 0x3f, 0x00, +}; + +XALIGNED(16) static const word32 L_mlkem_from_bytes_avx512_vbmi_mask_lo[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const word32 L_mlkem_from_bytes_avx512_vbmi_mask_hi[] + WC_X64I_UNUSED = { + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, +}; + +WC_X64I_TARGET("avx,avx512f,avx512vbmi") +WOLFSSL_LOCAL void mlkem_from_bytes_avx512_vbmi(sword16* p, const byte* b) +{ + word64 rdi, rsi; + __m512i z0, z1, z2, z3, z4, z5; + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)b; + + z0 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_from_bytes_avx512_vbmi_lo, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_from_bytes_avx512_vbmi_hi, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_from_bytes_avx512_vbmi_mask_lo, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_from_bytes_avx512_vbmi_mask_hi, 0)); + z4 = _mm512_permutexvar_epi8(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 0))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z4); + z4 = _mm512_permutexvar_epi8(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 48))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z4); + z4 = _mm512_permutexvar_epi8(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 96))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z4); + z4 = _mm512_permutexvar_epi8(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 144))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z4); + z4 = _mm512_permutexvar_epi8(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 192))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z4); + z4 = _mm512_permutexvar_epi8(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 240))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z4); + z4 = _mm512_permutexvar_epi8(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 288))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z4); + z4 = _mm512_permutexvar_epi8(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, + 320))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z4); +} + +XALIGNED(16) static const word32 L_mlkem_to_bytes_avx512_vbmi_mask[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const byte L_mlkem_to_bytes_avx512_vbmi_permb[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, + 0x0a, 0x0c, 0x0d, 0x0e, 0x10, 0x11, 0x12, 0x14, + 0x15, 0x16, 0x18, 0x19, 0x1a, 0x1c, 0x1d, 0x1e, + 0x20, 0x21, 0x22, 0x24, 0x25, 0x26, 0x28, 0x29, + 0x2a, 0x2c, 0x2d, 0x2e, 0x30, 0x31, 0x32, 0x34, + 0x35, 0x36, 0x38, 0x39, 0x3a, 0x3c, 0x3d, 0x3e, + 0x00, 0x01, 0x02, 0x04, 0x00, 0x01, 0x02, 0x04, + 0x00, 0x01, 0x02, 0x04, 0x00, 0x01, 0x02, 0x04, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi") +WOLFSSL_LOCAL void mlkem_to_bytes_avx512_vbmi(byte* b, sword16* p) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6; + + rdi = (word64)(size_t)b; + rsi = (word64)(size_t)p; + + rdx = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_to_bytes_avx512_vbmi_mask, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_to_bytes_avx512_vbmi_permb, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z4 = _mm512_sub_epi16(z3, z0); + z5 = _mm512_srai_epi16(z4, 15); + z5 = _mm512_and_si512(z5, z0); + z3 = _mm512_add_epi16(z5, z4); + z4 = _mm512_srli_epi32(z3, 16); + z4 = _mm512_slli_epi32(z4, 12); + z3 = _mm512_ternarylogic_epi32(z3, z1, z4, 0xea); + z6 = _mm512_permutexvar_epi8(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z6); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z4 = _mm512_sub_epi16(z3, z0); + z5 = _mm512_srai_epi16(z4, 15); + z5 = _mm512_and_si512(z5, z0); + z3 = _mm512_add_epi16(z5, z4); + z4 = _mm512_srli_epi32(z3, 16); + z4 = _mm512_slli_epi32(z4, 12); + z3 = _mm512_ternarylogic_epi32(z3, z1, z4, 0xea); + z6 = _mm512_permutexvar_epi8(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 48), z6); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z4 = _mm512_sub_epi16(z3, z0); + z5 = _mm512_srai_epi16(z4, 15); + z5 = _mm512_and_si512(z5, z0); + z3 = _mm512_add_epi16(z5, z4); + z4 = _mm512_srli_epi32(z3, 16); + z4 = _mm512_slli_epi32(z4, 12); + z3 = _mm512_ternarylogic_epi32(z3, z1, z4, 0xea); + z6 = _mm512_permutexvar_epi8(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 96), z6); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z4 = _mm512_sub_epi16(z3, z0); + z5 = _mm512_srai_epi16(z4, 15); + z5 = _mm512_and_si512(z5, z0); + z3 = _mm512_add_epi16(z5, z4); + z4 = _mm512_srli_epi32(z3, 16); + z4 = _mm512_slli_epi32(z4, 12); + z3 = _mm512_ternarylogic_epi32(z3, z1, z4, 0xea); + z6 = _mm512_permutexvar_epi8(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 144), z6); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z4 = _mm512_sub_epi16(z3, z0); + z5 = _mm512_srai_epi16(z4, 15); + z5 = _mm512_and_si512(z5, z0); + z3 = _mm512_add_epi16(z5, z4); + z4 = _mm512_srli_epi32(z3, 16); + z4 = _mm512_slli_epi32(z4, 12); + z3 = _mm512_ternarylogic_epi32(z3, z1, z4, 0xea); + z6 = _mm512_permutexvar_epi8(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z6); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z4 = _mm512_sub_epi16(z3, z0); + z5 = _mm512_srai_epi16(z4, 15); + z5 = _mm512_and_si512(z5, z0); + z3 = _mm512_add_epi16(z5, z4); + z4 = _mm512_srli_epi32(z3, 16); + z4 = _mm512_slli_epi32(z4, 12); + z3 = _mm512_ternarylogic_epi32(z3, z1, z4, 0xea); + z6 = _mm512_permutexvar_epi8(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 240), z6); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z4 = _mm512_sub_epi16(z3, z0); + z5 = _mm512_srai_epi16(z4, 15); + z5 = _mm512_and_si512(z5, z0); + z3 = _mm512_add_epi16(z5, z4); + z4 = _mm512_srli_epi32(z3, 16); + z4 = _mm512_slli_epi32(z4, 12); + z3 = _mm512_ternarylogic_epi32(z3, z1, z4, 0xea); + z6 = _mm512_permutexvar_epi8(z2, z3); + _mm512_storeu_si512((void*)WC_PW(rdi, 288), z6); + z3 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z4 = _mm512_sub_epi16(z3, z0); + z5 = _mm512_srai_epi16(z4, 15); + z5 = _mm512_and_si512(z5, z0); + z3 = _mm512_add_epi16(z5, z4); + z4 = _mm512_srli_epi32(z3, 16); + z4 = _mm512_slli_epi32(z4, 12); + z3 = _mm512_ternarylogic_epi32(z3, z1, z4, 0xea); + z6 = _mm512_permutexvar_epi8(z2, z3); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 336), _mm512_castsi512_si256(z6)); + z4 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z6, 2)); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 368), _mm512_castsi512_si128(z4)); +} + +XALIGNED(16) static const byte L_mlkem_rej_uniform_avx512_vbmi_permb[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x00, 0x03, 0x04, 0x05, 0x00, + 0x06, 0x07, 0x08, 0x00, 0x09, 0x0a, 0x0b, 0x00, + 0x0c, 0x0d, 0x0e, 0x00, 0x0f, 0x10, 0x11, 0x00, + 0x12, 0x13, 0x14, 0x00, 0x15, 0x16, 0x17, 0x00, + 0x18, 0x19, 0x1a, 0x00, 0x1b, 0x1c, 0x1d, 0x00, + 0x1e, 0x1f, 0x20, 0x00, 0x21, 0x22, 0x23, 0x00, + 0x24, 0x25, 0x26, 0x00, 0x27, 0x28, 0x29, 0x00, + 0x2a, 0x2b, 0x2c, 0x00, 0x2d, 0x2e, 0x2f, 0x00, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_avx512_vbmi_mask_lo[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_avx512_vbmi_mask_hi[] + WC_X64I_UNUSED = { + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi,bmi") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx512_vbmi(sword16* p, + unsigned int len, const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, rbx = 0, r11 = 0, r8 = 0, r9 = 0; + __m256i y8 = _mm256_setzero_si256(); + __m512i z0, z1, z2, z3, z4 = _mm512_setzero_si512(), + z5 = _mm512_setzero_si512(), z6 = _mm512_setzero_si512(), + z7 = _mm512_setzero_si512(); + __mmask32 k1, k2; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + rcx = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + r10 = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r10)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_avx512_vbmi_permb, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_avx512_vbmi_mask_lo, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_avx512_vbmi_mask_hi, 0)); +L_mlkem_rej_uniform_avx512_vbmi_fast: + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_avx512_vbmi_fast_end; + } + if ((sword32)((word32)rcx) < (sword32)(0x40)) { + goto L_mlkem_rej_uniform_avx512_vbmi_fast_end; + } + z4 = _mm512_permutexvar_epi8(z1, _mm512_loadu_si512((const void*)WC_PR(rdx, + 0))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + k1 = _mm512_cmp_epi16_mask(z4, z0, 1); + rbx = (word32)((word32)k1); + z6 = _mm512_cvtepu16_epi32(_mm512_castsi512_si256(z4)); + z7 = _mm512_mask_compress_epi32(z7, k1, z6); + z6 = _mm512_zextsi256_si512(_mm512_cvtepi32_epi16(z7)); + r11 = (word32)((word32)rbx); + r11 = (word32)((word32)r11 & 0xffff); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), _mm512_castsi512_si256(z6)); + y8 = _mm512_extracti64x4_epi64(z4, 1); + z6 = _mm512_cvtepu16_epi32(y8); + k2 = _kshiftri_mask32(k1, 16); + z7 = _mm512_mask_compress_epi32(z7, k2, z6); + z6 = _mm512_zextsi256_si512(_mm512_cvtepi32_epi16(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, r11 * 2), _mm512_castsi512_si256( + z6)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + rcx = (word32)((word32)rcx - 0x30); + goto L_mlkem_rej_uniform_avx512_vbmi_fast; +L_mlkem_rej_uniform_avx512_vbmi_fast_end: +L_mlkem_rej_uniform_avx512_vbmi_tail: + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_vbmi_tail_end; + } + if ((sword32)((word32)rcx) < (sword32)(3)) { + goto L_mlkem_rej_uniform_avx512_vbmi_tail_end; + } + r8 = (word32)((word32)WC_L16(rdx, 0)); + r8 = (word32)((word32)r8 & 0xfff); + r9 = (word32)((word32)WC_L16(rdx, 1)); + r9 = (word32)((word32)r9 >> 4); + if ((sword32)((word32)r8) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_avx512_vbmi_skip0; + } + WC_S16(rdi, 0) = (word16)((word16)r8); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_vbmi_tail_end; + } +L_mlkem_rej_uniform_avx512_vbmi_skip0: + if ((sword32)((word32)r9) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_avx512_vbmi_skip1; + } + WC_S16(rdi, 0) = (word16)((word16)r9); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_vbmi_tail_end; + } +L_mlkem_rej_uniform_avx512_vbmi_skip1: + rdx = (word64)(rdx + 3); + rcx = (word32)((word32)rcx - 3); + goto L_mlkem_rej_uniform_avx512_vbmi_tail; +L_mlkem_rej_uniform_avx512_vbmi_tail_end: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; +} + +XALIGNED(16) static const byte L_mlkem_rej_uniform_n_avx512_vbmi_permb[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x00, 0x03, 0x04, 0x05, 0x00, + 0x06, 0x07, 0x08, 0x00, 0x09, 0x0a, 0x0b, 0x00, + 0x0c, 0x0d, 0x0e, 0x00, 0x0f, 0x10, 0x11, 0x00, + 0x12, 0x13, 0x14, 0x00, 0x15, 0x16, 0x17, 0x00, + 0x18, 0x19, 0x1a, 0x00, 0x1b, 0x1c, 0x1d, 0x00, + 0x1e, 0x1f, 0x20, 0x00, 0x21, 0x22, 0x23, 0x00, + 0x24, 0x25, 0x26, 0x00, 0x27, 0x28, 0x29, 0x00, + 0x2a, 0x2b, 0x2c, 0x00, 0x2d, 0x2e, 0x2f, 0x00, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_n_avx512_vbmi_mask_lo[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_n_avx512_vbmi_mask_hi[] + WC_X64I_UNUSED = { + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi,bmi") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx512_vbmi(sword16* p, + unsigned int len, const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, rbx = 0, r11 = 0, r8 = 0, r9 = 0; + __m256i y8 = _mm256_setzero_si256(); + __m512i z0, z1, z2, z3, z4 = _mm512_setzero_si512(), + z5 = _mm512_setzero_si512(), z6 = _mm512_setzero_si512(), + z7 = _mm512_setzero_si512(); + __mmask32 k1, k2; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + rcx = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + r10 = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r10)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_vbmi_permb, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_vbmi_mask_lo, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_vbmi_mask_hi, 0)); +L_mlkem_rej_uniform_n_avx512_vbmi_fast: + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_fast_end; + } + if ((sword32)((word32)rcx) < (sword32)(0x40)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_fast_end; + } + z4 = _mm512_permutexvar_epi8(z1, _mm512_loadu_si512((const void*)WC_PR(rdx, + 0))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + k1 = _mm512_cmp_epi16_mask(z4, z0, 1); + rbx = (word32)((word32)k1); + z6 = _mm512_cvtepu16_epi32(_mm512_castsi512_si256(z4)); + z7 = _mm512_mask_compress_epi32(z7, k1, z6); + z6 = _mm512_zextsi256_si512(_mm512_cvtepi32_epi16(z7)); + r11 = (word32)((word32)rbx); + r11 = (word32)((word32)r11 & 0xffff); + r11 = (word32)(WC_X64I_POPCNT32((word32)r11)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), _mm512_castsi512_si256(z6)); + y8 = _mm512_extracti64x4_epi64(z4, 1); + z6 = _mm512_cvtepu16_epi32(y8); + k2 = _kshiftri_mask32(k1, 16); + z7 = _mm512_mask_compress_epi32(z7, k2, z6); + z6 = _mm512_zextsi256_si512(_mm512_cvtepi32_epi16(z7)); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, r11 * 2), _mm512_castsi512_si256( + z6)); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + rcx = (word32)((word32)rcx - 0x30); + goto L_mlkem_rej_uniform_n_avx512_vbmi_fast; +L_mlkem_rej_uniform_n_avx512_vbmi_fast_end: +L_mlkem_rej_uniform_n_avx512_vbmi_tail: + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_tail_end; + } + if ((sword32)((word32)rcx) < (sword32)(3)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_tail_end; + } + r8 = (word32)((word32)WC_L16(rdx, 0)); + r8 = (word32)((word32)r8 & 0xfff); + r9 = (word32)((word32)WC_L16(rdx, 1)); + r9 = (word32)((word32)r9 >> 4); + if ((sword32)((word32)r8) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_skip0; + } + WC_S16(rdi, 0) = (word16)((word16)r8); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_tail_end; + } +L_mlkem_rej_uniform_n_avx512_vbmi_skip0: + if ((sword32)((word32)r9) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_skip1; + } + WC_S16(rdi, 0) = (word16)((word16)r9); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_tail_end; + } +L_mlkem_rej_uniform_n_avx512_vbmi_skip1: + rdx = (word64)(rdx + 3); + rcx = (word32)((word32)rcx - 3); + goto L_mlkem_rej_uniform_n_avx512_vbmi_tail; +L_mlkem_rej_uniform_n_avx512_vbmi_tail_end: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; +} + +XALIGNED(16) static const byte L_mlkem_compress_4_avx512_vbmi_gather[] + WC_X64I_UNUSED = { + 0x00, 0x04, 0x08, 0x0c, 0x10, 0x14, 0x18, 0x1c, + 0x20, 0x24, 0x28, 0x2c, 0x30, 0x34, 0x38, 0x3c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi") +WOLFSSL_LOCAL void mlkem_compress_4_avx512_vbmi(byte* b, const sword16* p) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6; + + rdi = (word64)(size_t)b; + rsi = (word64)(size_t)p; + + z0 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_compress_4_avx512_vbmi_gather, 0)); + rdx = (word64)(0x4ebf); + z1 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z1 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z1)); + rdx = (word64)(0x200); + z2 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z2 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z2)); + rdx = (word64)(0xf); + z3 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z3 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z3)); + z4 = _mm512_mulhi_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, 0))); + z4 = _mm512_mulhrs_epi16(z4, z2); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_srli_epi32(z4, 12); + z4 = _mm512_or_si512(z4, z5); + z6 = _mm512_permutexvar_epi8(z0, z4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm512_castsi512_si128(z6)); + z4 = _mm512_mulhi_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, + 64))); + z4 = _mm512_mulhrs_epi16(z4, z2); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_srli_epi32(z4, 12); + z4 = _mm512_or_si512(z4, z5); + z6 = _mm512_permutexvar_epi8(z0, z4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 16), _mm512_castsi512_si128(z6)); + z4 = _mm512_mulhi_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, + 128))); + z4 = _mm512_mulhrs_epi16(z4, z2); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_srli_epi32(z4, 12); + z4 = _mm512_or_si512(z4, z5); + z6 = _mm512_permutexvar_epi8(z0, z4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 32), _mm512_castsi512_si128(z6)); + z4 = _mm512_mulhi_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, + 192))); + z4 = _mm512_mulhrs_epi16(z4, z2); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_srli_epi32(z4, 12); + z4 = _mm512_or_si512(z4, z5); + z6 = _mm512_permutexvar_epi8(z0, z4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 48), _mm512_castsi512_si128(z6)); + z4 = _mm512_mulhi_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, + 256))); + z4 = _mm512_mulhrs_epi16(z4, z2); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_srli_epi32(z4, 12); + z4 = _mm512_or_si512(z4, z5); + z6 = _mm512_permutexvar_epi8(z0, z4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 64), _mm512_castsi512_si128(z6)); + z4 = _mm512_mulhi_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, + 320))); + z4 = _mm512_mulhrs_epi16(z4, z2); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_srli_epi32(z4, 12); + z4 = _mm512_or_si512(z4, z5); + z6 = _mm512_permutexvar_epi8(z0, z4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), _mm512_castsi512_si128(z6)); + z4 = _mm512_mulhi_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, + 384))); + z4 = _mm512_mulhrs_epi16(z4, z2); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_srli_epi32(z4, 12); + z4 = _mm512_or_si512(z4, z5); + z6 = _mm512_permutexvar_epi8(z0, z4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 96), _mm512_castsi512_si128(z6)); + z4 = _mm512_mulhi_epi16(z1, _mm512_loadu_si512((const void*)WC_PR(rsi, + 448))); + z4 = _mm512_mulhrs_epi16(z4, z2); + z4 = _mm512_and_si512(z4, z3); + z5 = _mm512_srli_epi32(z4, 12); + z4 = _mm512_or_si512(z4, z5); + z6 = _mm512_permutexvar_epi8(z0, z4); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 112), _mm512_castsi512_si128(z6)); +} + +XALIGNED(16) static const word16 L_mlkem_compress_5_avx512_vbmi_shift1[] + WC_X64I_UNUSED = { + 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, + 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, 0x2001, +}; + +XALIGNED(16) static const word32 L_mlkem_compress_5_avx512_vbmi_shift2[] + WC_X64I_UNUSED = { + 0x04000001, 0x04000001, 0x04000001, 0x04000001, + 0x04000001, 0x04000001, 0x04000001, 0x04000001, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_5_avx512_vbmi_shlv[] + WC_X64I_UNUSED = { + 0x000000000000000cULL, 0x000000000000000cULL, + 0x000000000000000cULL, 0x000000000000000cULL, +}; + +XALIGNED(16) static const byte L_mlkem_compress_5_avx512_vbmi_gather[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x10, 0x11, 0x12, 0x13, 0x14, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +WC_X64I_TARGET("avx512f,avx512vl,avx512bw,avx512vbmi") +WOLFSSL_LOCAL void mlkem_compress_5_avx512_vbmi(byte* b, const sword16* p) +{ + word64 rdi, rsi, rdx; + __m256i y3, y4, y5, y8, y9; + __m512i z0, z1, z2, z6, z7; + + rdi = (word64)(size_t)b; + rsi = (word64)(size_t)p; + + rdx = (word64)(0x4ebf); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + rdx = (word64)(0x400); + z1 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z1 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z1)); + rdx = (word64)(0x1f); + z2 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rdx)); + z2 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z2)); + y3 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx512_vbmi_shift1, 0)); + y4 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx512_vbmi_shift2, 0)); + y5 = _mm256_loadu_si256((const __m256i*)WC_PR( + L_mlkem_compress_5_avx512_vbmi_shlv, 0)); + z6 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_compress_5_avx512_vbmi_gather, 0)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, 0))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y8 = _mm512_cvtepi16_epi8(z7); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srli_epi64(y8, 12); + y9 = _mm256_permutexvar_epi8(_mm512_castsi512_si256(z6), y8); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 0), _mm256_castsi256_si128(y9)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y9, 1)); + WC_S32(rdi, 16) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y8)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 64))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y8 = _mm512_cvtepi16_epi8(z7); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srli_epi64(y8, 12); + y9 = _mm256_permutexvar_epi8(_mm512_castsi512_si256(z6), y8); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 20), _mm256_castsi256_si128(y9)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y9, 1)); + WC_S32(rdi, 36) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y8)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 128))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y8 = _mm512_cvtepi16_epi8(z7); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srli_epi64(y8, 12); + y9 = _mm256_permutexvar_epi8(_mm512_castsi512_si256(z6), y8); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 40), _mm256_castsi256_si128(y9)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y9, 1)); + WC_S32(rdi, 56) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y8)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 192))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y8 = _mm512_cvtepi16_epi8(z7); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srli_epi64(y8, 12); + y9 = _mm256_permutexvar_epi8(_mm512_castsi512_si256(z6), y8); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 60), _mm256_castsi256_si128(y9)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y9, 1)); + WC_S32(rdi, 76) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y8)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 256))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y8 = _mm512_cvtepi16_epi8(z7); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srli_epi64(y8, 12); + y9 = _mm256_permutexvar_epi8(_mm512_castsi512_si256(z6), y8); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 80), _mm256_castsi256_si128(y9)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y9, 1)); + WC_S32(rdi, 96) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y8)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 320))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y8 = _mm512_cvtepi16_epi8(z7); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srli_epi64(y8, 12); + y9 = _mm256_permutexvar_epi8(_mm512_castsi512_si256(z6), y8); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 100), _mm256_castsi256_si128(y9)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y9, 1)); + WC_S32(rdi, 116) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y8)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 384))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y8 = _mm512_cvtepi16_epi8(z7); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srli_epi64(y8, 12); + y9 = _mm256_permutexvar_epi8(_mm512_castsi512_si256(z6), y8); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 120), _mm256_castsi256_si128(y9)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y9, 1)); + WC_S32(rdi, 136) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y8)); + z7 = _mm512_mulhi_epi16(z0, _mm512_loadu_si512((const void*)WC_PR(rsi, + 448))); + z7 = _mm512_mulhrs_epi16(z7, z1); + z7 = _mm512_and_si512(z7, z2); + y8 = _mm512_cvtepi16_epi8(z7); + y8 = _mm256_maddubs_epi16(y8, y3); + y8 = _mm256_madd_epi16(y8, y4); + y8 = _mm256_sllv_epi32(y8, y5); + y8 = _mm256_srli_epi64(y8, 12); + y9 = _mm256_permutexvar_epi8(_mm512_castsi512_si256(z6), y8); + _mm_storeu_si128((__m128i*)WC_PW(rdi, 140), _mm256_castsi256_si128(y9)); + y8 = _mm256_zextsi128_si256(_mm256_extracti128_si256(y9, 1)); + WC_S32(rdi, 156) = (word32)_mm_cvtsi128_si32(_mm256_castsi256_si128(y8)); +} + +XALIGNED(32) static const word64 L_mlkem_compress_10_avx512_vbmi_shift[] + WC_X64I_UNUSED = { + 0x0400000104000001ULL, 0x0400000104000001ULL, + 0x0400000104000001ULL, 0x0400000104000001ULL, + 0x0400000104000001ULL, 0x0400000104000001ULL, + 0x0400000104000001ULL, 0x0400000104000001ULL, +}; + +XALIGNED(32) static const word64 L_mlkem_compress_10_avx512_vbmi_shlv[] + WC_X64I_UNUSED = { + 0x000000000000000cULL, 0x000000000000000cULL, + 0x000000000000000cULL, 0x000000000000000cULL, + 0x000000000000000cULL, 0x000000000000000cULL, + 0x000000000000000cULL, 0x000000000000000cULL, +}; + +XALIGNED(16) static const byte L_mlkem_compress_10_avx512_vbmi_gather[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, + 0x0b, 0x0c, 0x10, 0x11, 0x12, 0x13, 0x14, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x30, 0x31, + 0x32, 0x33, 0x34, 0x38, 0x39, 0x3a, 0x3b, 0x3c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi") +WOLFSSL_LOCAL void mlkem_compress_10_avx512_vbmi(byte* r, const sword16* p, + int n) +{ + word64 rdi, rsi, rdx, rax; + __m512i z0, z1, z2, z3, z4, z5, z6, z7, z8 = _mm512_setzero_si512(), + z9 = _mm512_setzero_si512(), z10 = _mm512_setzero_si512(), + z11 = _mm512_setzero_si512(), z12 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)r; + rsi = (word64)(size_t)p; + rdx = (word64)(word32)n; + + rax = (word64)(0x4ebf); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + rax = (word64)(0xf); + z2 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z2 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z2)); + rax = (word64)(0x1000); + z3 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z3 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z3)); + rax = (word64)(0x3ff); + z4 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)rax)); + z4 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z4)); + z1 = _mm512_slli_epi16(z0, 3); + z5 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_compress_10_avx512_vbmi_shift, 0)); + z6 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_compress_10_avx512_vbmi_shlv, 0)); + z7 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_compress_10_avx512_vbmi_gather, 0)); +L_mlkem_compress_10_avx512_vbmi_start: + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 0)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + z11 = _mm512_madd_epi16(z8, z5); + z11 = _mm512_sllv_epi32(z11, z6); + z11 = _mm512_srli_epi64(z11, 12); + z12 = _mm512_permutexvar_epi8(z7, z11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 0), _mm512_castsi512_si256(z12)); + z9 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z12, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 32), _mm512_castsi512_si128(z9)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 64)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + z11 = _mm512_madd_epi16(z8, z5); + z11 = _mm512_sllv_epi32(z11, z6); + z11 = _mm512_srli_epi64(z11, 12); + z12 = _mm512_permutexvar_epi8(z7, z11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 40), _mm512_castsi512_si256(z12)); + z9 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z12, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 72), _mm512_castsi512_si128(z9)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 128)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + z11 = _mm512_madd_epi16(z8, z5); + z11 = _mm512_sllv_epi32(z11, z6); + z11 = _mm512_srli_epi64(z11, 12); + z12 = _mm512_permutexvar_epi8(z7, z11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 80), _mm512_castsi512_si256(z12)); + z9 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z12, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 112), _mm512_castsi512_si128(z9)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 192)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + z11 = _mm512_madd_epi16(z8, z5); + z11 = _mm512_sllv_epi32(z11, z6); + z11 = _mm512_srli_epi64(z11, 12); + z12 = _mm512_permutexvar_epi8(z7, z11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 120), _mm512_castsi512_si256(z12)); + z9 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z12, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 152), _mm512_castsi512_si128(z9)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 256)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + z11 = _mm512_madd_epi16(z8, z5); + z11 = _mm512_sllv_epi32(z11, z6); + z11 = _mm512_srli_epi64(z11, 12); + z12 = _mm512_permutexvar_epi8(z7, z11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 160), _mm512_castsi512_si256(z12)); + z9 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z12, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 192), _mm512_castsi512_si128(z9)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 320)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + z11 = _mm512_madd_epi16(z8, z5); + z11 = _mm512_sllv_epi32(z11, z6); + z11 = _mm512_srli_epi64(z11, 12); + z12 = _mm512_permutexvar_epi8(z7, z11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 200), _mm512_castsi512_si256(z12)); + z9 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z12, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 232), _mm512_castsi512_si128(z9)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 384)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + z11 = _mm512_madd_epi16(z8, z5); + z11 = _mm512_sllv_epi32(z11, z6); + z11 = _mm512_srli_epi64(z11, 12); + z12 = _mm512_permutexvar_epi8(z7, z11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 240), _mm512_castsi512_si256(z12)); + z9 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z12, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 272), _mm512_castsi512_si128(z9)); + z8 = _mm512_loadu_si512((const void*)WC_PR(rsi, 448)); + z9 = _mm512_mullo_epi16(z8, z1); + z10 = _mm512_add_epi16(z8, z2); + z8 = _mm512_slli_epi16(z8, 3); + z8 = _mm512_mulhi_epu16(z8, z0); + z10 = _mm512_sub_epi16(z9, z10); + z9 = _mm512_andnot_si512(z9, z10); + z9 = _mm512_srli_epi16(z9, 15); + z8 = _mm512_sub_epi16(z8, z9); + z8 = _mm512_mulhrs_epi16(z8, z3); + z8 = _mm512_and_si512(z8, z4); + z11 = _mm512_madd_epi16(z8, z5); + z11 = _mm512_sllv_epi32(z11, z6); + z11 = _mm512_srli_epi64(z11, 12); + z12 = _mm512_permutexvar_epi8(z7, z11); + _mm256_storeu_si256((__m256i*)WC_PW(rdi, 280), _mm512_castsi512_si256(z12)); + z9 = _mm512_zextsi128_si512(_mm512_extracti32x4_epi32(z12, 2)); + _mm_storel_epi64((__m128i*)WC_PW(rdi, 312), _mm512_castsi512_si128(z9)); + rdi = (word64)(rdi + 0x140); + rsi = (word64)(rsi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_compress_10_avx512_vbmi_start; + } +} + +XALIGNED(16) static const word32 L_mlkem_decompress_10_avx512_vbmi_mask[] + WC_X64I_UNUSED = { + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, + 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, 0x7fe01ff8, +}; + +XALIGNED(32) static const word64 L_mlkem_decompress_10_avx512_vbmi_sllv[] + WC_X64I_UNUSED = { + 0x0000000000000004ULL, 0x0000000000000004ULL, + 0x0000000000000004ULL, 0x0000000000000004ULL, + 0x0000000000000004ULL, 0x0000000000000004ULL, + 0x0000000000000004ULL, 0x0000000000000004ULL, +}; + +XALIGNED(16) static const word32 L_mlkem_decompress_10_avx512_vbmi_q[] + WC_X64I_UNUSED = { + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, + 0x0d013404, 0x0d013404, 0x0d013404, 0x0d013404, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_10_avx512_vbmi_lo[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x04, + 0x05, 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, + 0x0a, 0x0b, 0x0b, 0x0c, 0x0c, 0x0d, 0x0d, 0x0e, + 0x0f, 0x10, 0x10, 0x11, 0x11, 0x12, 0x12, 0x13, + 0x14, 0x15, 0x15, 0x16, 0x16, 0x17, 0x17, 0x18, + 0x19, 0x1a, 0x1a, 0x1b, 0x1b, 0x1c, 0x1c, 0x1d, + 0x1e, 0x1f, 0x1f, 0x20, 0x20, 0x21, 0x21, 0x22, + 0x23, 0x24, 0x24, 0x25, 0x25, 0x26, 0x26, 0x27, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_10_avx512_vbmi_hi[] + WC_X64I_UNUSED = { + 0x18, 0x19, 0x19, 0x1a, 0x1a, 0x1b, 0x1b, 0x1c, + 0x1d, 0x1e, 0x1e, 0x1f, 0x1f, 0x20, 0x20, 0x21, + 0x22, 0x23, 0x23, 0x24, 0x24, 0x25, 0x25, 0x26, + 0x27, 0x28, 0x28, 0x29, 0x29, 0x2a, 0x2a, 0x2b, + 0x2c, 0x2d, 0x2d, 0x2e, 0x2e, 0x2f, 0x2f, 0x30, + 0x31, 0x32, 0x32, 0x33, 0x33, 0x34, 0x34, 0x35, + 0x36, 0x37, 0x37, 0x38, 0x38, 0x39, 0x39, 0x3a, + 0x3b, 0x3c, 0x3c, 0x3d, 0x3d, 0x3e, 0x3e, 0x3f, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi") +WOLFSSL_LOCAL void mlkem_decompress_10_avx512_vbmi(sword16* p, const byte* r, + int n) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + rdx = (word64)(word32)n; + + z0 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_10_avx512_vbmi_mask, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_10_avx512_vbmi_q, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_10_avx512_vbmi_sllv, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_10_avx512_vbmi_lo, 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_10_avx512_vbmi_hi, 0)); +L_mlkem_decompress_10_avx512_vbmi_start: + z5 = _mm512_permutexvar_epi8(z3, _mm512_loadu_si512((const void*)WC_PR(rsi, + 0))); + z5 = _mm512_sllv_epi32(z5, z2); + z5 = _mm512_srli_epi16(z5, 1); + z5 = _mm512_and_si512(z5, z0); + z5 = _mm512_mulhrs_epi16(z5, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z5); + z5 = _mm512_permutexvar_epi8(z3, _mm512_loadu_si512((const void*)WC_PR(rsi, + 40))); + z5 = _mm512_sllv_epi32(z5, z2); + z5 = _mm512_srli_epi16(z5, 1); + z5 = _mm512_and_si512(z5, z0); + z5 = _mm512_mulhrs_epi16(z5, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z5); + z5 = _mm512_permutexvar_epi8(z3, _mm512_loadu_si512((const void*)WC_PR(rsi, + 80))); + z5 = _mm512_sllv_epi32(z5, z2); + z5 = _mm512_srli_epi16(z5, 1); + z5 = _mm512_and_si512(z5, z0); + z5 = _mm512_mulhrs_epi16(z5, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z5); + z5 = _mm512_permutexvar_epi8(z3, _mm512_loadu_si512((const void*)WC_PR(rsi, + 120))); + z5 = _mm512_sllv_epi32(z5, z2); + z5 = _mm512_srli_epi16(z5, 1); + z5 = _mm512_and_si512(z5, z0); + z5 = _mm512_mulhrs_epi16(z5, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z5); + z5 = _mm512_permutexvar_epi8(z3, _mm512_loadu_si512((const void*)WC_PR(rsi, + 160))); + z5 = _mm512_sllv_epi32(z5, z2); + z5 = _mm512_srli_epi16(z5, 1); + z5 = _mm512_and_si512(z5, z0); + z5 = _mm512_mulhrs_epi16(z5, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z5); + z5 = _mm512_permutexvar_epi8(z3, _mm512_loadu_si512((const void*)WC_PR(rsi, + 200))); + z5 = _mm512_sllv_epi32(z5, z2); + z5 = _mm512_srli_epi16(z5, 1); + z5 = _mm512_and_si512(z5, z0); + z5 = _mm512_mulhrs_epi16(z5, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z5); + z5 = _mm512_permutexvar_epi8(z3, _mm512_loadu_si512((const void*)WC_PR(rsi, + 240))); + z5 = _mm512_sllv_epi32(z5, z2); + z5 = _mm512_srli_epi16(z5, 1); + z5 = _mm512_and_si512(z5, z0); + z5 = _mm512_mulhrs_epi16(z5, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z5); + z5 = _mm512_permutexvar_epi8(z4, _mm512_loadu_si512((const void*)WC_PR(rsi, + 256))); + z5 = _mm512_sllv_epi32(z5, z2); + z5 = _mm512_srli_epi16(z5, 1); + z5 = _mm512_and_si512(z5, z0); + z5 = _mm512_mulhrs_epi16(z5, z1); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z5); + rsi = (word64)(rsi + 0x140); + rdi = (word64)(rdi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_decompress_10_avx512_vbmi_start; + } +} + +XALIGNED(16) static const word16 L_mlkem_decompress_11_avx512_vbmi_q[] + WC_X64I_UNUSED = { + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, + 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, +}; + +XALIGNED(16) static const word32 L_mlkem_decompress_11_avx512_vbmi_srlvd[] + WC_X64I_UNUSED = { + 0x00000000, 0x00000001, 0x00000000, 0x00000000, + 0x00000000, 0x00000001, 0x00000000, 0x00000000, + 0x00000000, 0x00000001, 0x00000000, 0x00000000, + 0x00000000, 0x00000001, 0x00000000, 0x00000000, +}; + +XALIGNED(32) static const word64 L_mlkem_decompress_11_avx512_vbmi_srlvq[] + WC_X64I_UNUSED = { + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_11_avx512_vbmi_shift[] + WC_X64I_UNUSED = { + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, + 0x0020, 0x0004, 0x0001, 0x0020, 0x0008, 0x0001, 0x0020, 0x0004, +}; + +XALIGNED(16) static const word16 L_mlkem_decompress_11_avx512_vbmi_mask[] + WC_X64I_UNUSED = { + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, + 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, 0x7ff0, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_11_avx512_vbmi_lo[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x04, 0x05, + 0x05, 0x06, 0x06, 0x07, 0x08, 0x09, 0x09, 0x0a, + 0x0b, 0x0c, 0x0c, 0x0d, 0x0d, 0x0e, 0x0f, 0x10, + 0x10, 0x11, 0x11, 0x12, 0x13, 0x14, 0x14, 0x15, + 0x16, 0x17, 0x17, 0x18, 0x18, 0x19, 0x1a, 0x1b, + 0x1b, 0x1c, 0x1c, 0x1d, 0x1e, 0x1f, 0x1f, 0x20, + 0x21, 0x22, 0x22, 0x23, 0x23, 0x24, 0x25, 0x26, + 0x26, 0x27, 0x27, 0x28, 0x29, 0x2a, 0x2a, 0x2b, +}; + +XALIGNED(16) static const byte L_mlkem_decompress_11_avx512_vbmi_hi[] + WC_X64I_UNUSED = { + 0x14, 0x15, 0x15, 0x16, 0x16, 0x17, 0x18, 0x19, + 0x19, 0x1a, 0x1a, 0x1b, 0x1c, 0x1d, 0x1d, 0x1e, + 0x1f, 0x20, 0x20, 0x21, 0x21, 0x22, 0x23, 0x24, + 0x24, 0x25, 0x25, 0x26, 0x27, 0x28, 0x28, 0x29, + 0x2a, 0x2b, 0x2b, 0x2c, 0x2c, 0x2d, 0x2e, 0x2f, + 0x2f, 0x30, 0x30, 0x31, 0x32, 0x33, 0x33, 0x34, + 0x35, 0x36, 0x36, 0x37, 0x37, 0x38, 0x39, 0x3a, + 0x3a, 0x3b, 0x3b, 0x3c, 0x3d, 0x3e, 0x3e, 0x3f, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi") +WOLFSSL_LOCAL void mlkem_decompress_11_avx512_vbmi(sword16* p, const byte* r, + int n) +{ + word64 rdi, rsi, rdx; + __m512i z0, z1, z2, z3, z4, z5, z6, z7 = _mm512_setzero_si512(); + + rdi = (word64)(size_t)p; + rsi = (word64)(size_t)r; + rdx = (word64)(word32)n; + + z0 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_vbmi_q, 0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_vbmi_srlvd, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_vbmi_srlvq, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_vbmi_shift, 0)); + z4 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_vbmi_mask, 0)); + z5 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_vbmi_lo, 0)); + z6 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_decompress_11_avx512_vbmi_hi, 0)); +L_mlkem_decompress_11_avx512_vbmi_start: + z7 = _mm512_permutexvar_epi8(z5, _mm512_loadu_si512((const void*)WC_PR(rsi, + 0))); + z7 = _mm512_srlv_epi32(z7, z1); + z7 = _mm512_srlv_epi64(z7, z2); + z7 = _mm512_mullo_epi16(z7, z3); + z7 = _mm512_srli_epi16(z7, 1); + z7 = _mm512_and_si512(z7, z4); + z7 = _mm512_mulhrs_epi16(z7, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 0), z7); + z7 = _mm512_permutexvar_epi8(z5, _mm512_loadu_si512((const void*)WC_PR(rsi, + 44))); + z7 = _mm512_srlv_epi32(z7, z1); + z7 = _mm512_srlv_epi64(z7, z2); + z7 = _mm512_mullo_epi16(z7, z3); + z7 = _mm512_srli_epi16(z7, 1); + z7 = _mm512_and_si512(z7, z4); + z7 = _mm512_mulhrs_epi16(z7, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 64), z7); + z7 = _mm512_permutexvar_epi8(z5, _mm512_loadu_si512((const void*)WC_PR(rsi, + 88))); + z7 = _mm512_srlv_epi32(z7, z1); + z7 = _mm512_srlv_epi64(z7, z2); + z7 = _mm512_mullo_epi16(z7, z3); + z7 = _mm512_srli_epi16(z7, 1); + z7 = _mm512_and_si512(z7, z4); + z7 = _mm512_mulhrs_epi16(z7, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 128), z7); + z7 = _mm512_permutexvar_epi8(z5, _mm512_loadu_si512((const void*)WC_PR(rsi, + 132))); + z7 = _mm512_srlv_epi32(z7, z1); + z7 = _mm512_srlv_epi64(z7, z2); + z7 = _mm512_mullo_epi16(z7, z3); + z7 = _mm512_srli_epi16(z7, 1); + z7 = _mm512_and_si512(z7, z4); + z7 = _mm512_mulhrs_epi16(z7, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 192), z7); + z7 = _mm512_permutexvar_epi8(z5, _mm512_loadu_si512((const void*)WC_PR(rsi, + 176))); + z7 = _mm512_srlv_epi32(z7, z1); + z7 = _mm512_srlv_epi64(z7, z2); + z7 = _mm512_mullo_epi16(z7, z3); + z7 = _mm512_srli_epi16(z7, 1); + z7 = _mm512_and_si512(z7, z4); + z7 = _mm512_mulhrs_epi16(z7, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 256), z7); + z7 = _mm512_permutexvar_epi8(z5, _mm512_loadu_si512((const void*)WC_PR(rsi, + 220))); + z7 = _mm512_srlv_epi32(z7, z1); + z7 = _mm512_srlv_epi64(z7, z2); + z7 = _mm512_mullo_epi16(z7, z3); + z7 = _mm512_srli_epi16(z7, 1); + z7 = _mm512_and_si512(z7, z4); + z7 = _mm512_mulhrs_epi16(z7, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 320), z7); + z7 = _mm512_permutexvar_epi8(z5, _mm512_loadu_si512((const void*)WC_PR(rsi, + 264))); + z7 = _mm512_srlv_epi32(z7, z1); + z7 = _mm512_srlv_epi64(z7, z2); + z7 = _mm512_mullo_epi16(z7, z3); + z7 = _mm512_srli_epi16(z7, 1); + z7 = _mm512_and_si512(z7, z4); + z7 = _mm512_mulhrs_epi16(z7, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 384), z7); + z7 = _mm512_permutexvar_epi8(z6, _mm512_loadu_si512((const void*)WC_PR(rsi, + 288))); + z7 = _mm512_srlv_epi32(z7, z1); + z7 = _mm512_srlv_epi64(z7, z2); + z7 = _mm512_mullo_epi16(z7, z3); + z7 = _mm512_srli_epi16(z7, 1); + z7 = _mm512_and_si512(z7, z4); + z7 = _mm512_mulhrs_epi16(z7, z0); + _mm512_storeu_si512((void*)WC_PW(rdi, 448), z7); + rsi = (word64)(rsi + 0x160); + rdi = (word64)(rdi + 0x200); + rdx = (word32)((word32)rdx - 1); + if ((sword32)((word32)rdx) > (sword32)(0)) { + goto L_mlkem_decompress_11_avx512_vbmi_start; + } +} + +#ifdef HAVE_INTEL_AVX512_VBMI2 +XALIGNED(16) static const byte L_mlkem_rej_uniform_avx512_vbmi_vbmi2_permb[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x00, 0x03, 0x04, 0x05, 0x00, + 0x06, 0x07, 0x08, 0x00, 0x09, 0x0a, 0x0b, 0x00, + 0x0c, 0x0d, 0x0e, 0x00, 0x0f, 0x10, 0x11, 0x00, + 0x12, 0x13, 0x14, 0x00, 0x15, 0x16, 0x17, 0x00, + 0x18, 0x19, 0x1a, 0x00, 0x1b, 0x1c, 0x1d, 0x00, + 0x1e, 0x1f, 0x20, 0x00, 0x21, 0x22, 0x23, 0x00, + 0x24, 0x25, 0x26, 0x00, 0x27, 0x28, 0x29, 0x00, + 0x2a, 0x2b, 0x2c, 0x00, 0x2d, 0x2e, 0x2f, 0x00, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_avx512_vbmi_vbmi2_mask_lo[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_avx512_vbmi_vbmi2_mask_hi[] + WC_X64I_UNUSED = { + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi,avx512vbmi2,bmi") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_avx512_vbmi_vbmi2(sword16* p, + unsigned int len, const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, rbx = 0, r8 = 0, r9 = 0; + __m512i z0, z1, z2, z3, z4 = _mm512_setzero_si512(), + z5 = _mm512_setzero_si512(); + __mmask32 k1; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + rcx = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + r10 = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r10)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_avx512_vbmi_vbmi2_permb, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_avx512_vbmi_vbmi2_mask_lo, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_avx512_vbmi_vbmi2_mask_hi, 0)); +L_mlkem_rej_uniform_avx512_vbmi_vbmi2_fast: + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_fast_end; + } + if ((sword32)((word32)rcx) < (sword32)(0x40)) { + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_fast_end; + } + z4 = _mm512_permutexvar_epi8(z1, _mm512_loadu_si512((const void*)WC_PR(rdx, + 0))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + k1 = _mm512_cmp_epi16_mask(z4, z0, 1); + _mm512_mask_compressstoreu_epi16(WC_PW(rdi, 0), k1, z4); + rbx = (word32)((word32)k1); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + rcx = (word32)((word32)rcx - 0x30); + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_fast; +L_mlkem_rej_uniform_avx512_vbmi_vbmi2_fast_end: +L_mlkem_rej_uniform_avx512_vbmi_vbmi2_tail: + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_tail_end; + } + if ((sword32)((word32)rcx) < (sword32)(3)) { + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_tail_end; + } + r8 = (word32)((word32)WC_L16(rdx, 0)); + r8 = (word32)((word32)r8 & 0xfff); + r9 = (word32)((word32)WC_L16(rdx, 1)); + r9 = (word32)((word32)r9 >> 4); + if ((sword32)((word32)r8) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_skip0; + } + WC_S16(rdi, 0) = (word16)((word16)r8); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_tail_end; + } +L_mlkem_rej_uniform_avx512_vbmi_vbmi2_skip0: + if ((sword32)((word32)r9) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_skip1; + } + WC_S16(rdi, 0) = (word16)((word16)r9); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_tail_end; + } +L_mlkem_rej_uniform_avx512_vbmi_vbmi2_skip1: + rdx = (word64)(rdx + 3); + rcx = (word32)((word32)rcx - 3); + goto L_mlkem_rej_uniform_avx512_vbmi_vbmi2_tail; +L_mlkem_rej_uniform_avx512_vbmi_vbmi2_tail_end: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; +} + +XALIGNED(16) static const byte L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_permb[] + WC_X64I_UNUSED = { + 0x00, 0x01, 0x02, 0x00, 0x03, 0x04, 0x05, 0x00, + 0x06, 0x07, 0x08, 0x00, 0x09, 0x0a, 0x0b, 0x00, + 0x0c, 0x0d, 0x0e, 0x00, 0x0f, 0x10, 0x11, 0x00, + 0x12, 0x13, 0x14, 0x00, 0x15, 0x16, 0x17, 0x00, + 0x18, 0x19, 0x1a, 0x00, 0x1b, 0x1c, 0x1d, 0x00, + 0x1e, 0x1f, 0x20, 0x00, 0x21, 0x22, 0x23, 0x00, + 0x24, 0x25, 0x26, 0x00, 0x27, 0x28, 0x29, 0x00, + 0x2a, 0x2b, 0x2c, 0x00, 0x2d, 0x2e, 0x2f, 0x00, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_mask_lo[] + WC_X64I_UNUSED = { + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, + 0x00000fff, 0x00000fff, 0x00000fff, 0x00000fff, +}; + +XALIGNED(16) static const word32 L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_mask_hi[] + WC_X64I_UNUSED = { + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, + 0x00fff000, 0x00fff000, 0x00fff000, 0x00fff000, +}; + +WC_X64I_TARGET("avx512f,avx512bw,avx512vbmi,avx512vbmi2,bmi") +WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_n_avx512_vbmi_vbmi2(sword16* p, + unsigned int len, const byte* r, unsigned int rLen) +{ + word64 rdi, rsi, rdx, rcx, rax, r10, rbx = 0, r8 = 0, r9 = 0; + __m512i z0, z1, z2, z3, z4 = _mm512_setzero_si512(), + z5 = _mm512_setzero_si512(); + __mmask32 k1; + + rdi = (word64)(size_t)p; + rsi = (word64)(word32)len; + rdx = (word64)(size_t)r; + rcx = (word64)(word32)rLen; + + rax = (word32)((word32)rsi); + r10 = (word64)(0xd01); + z0 = _mm512_zextsi128_si512(_mm_cvtsi32_si128((int)(word32)r10)); + z0 = _mm512_broadcastw_epi16(_mm512_castsi512_si128(z0)); + z1 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_permb, 0)); + z2 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_mask_lo, 0)); + z3 = _mm512_loadu_si512((const void*)WC_PR( + L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_mask_hi, 0)); +L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_fast: + if ((sword32)((word32)rsi) < (sword32)(0x20)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_fast_end; + } + if ((sword32)((word32)rcx) < (sword32)(0x40)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_fast_end; + } + z4 = _mm512_permutexvar_epi8(z1, _mm512_loadu_si512((const void*)WC_PR(rdx, + 0))); + z5 = _mm512_and_si512(z4, z3); + z5 = _mm512_slli_epi32(z5, 4); + z4 = _mm512_ternarylogic_epi32(z4, z2, z5, 0xea); + k1 = _mm512_cmp_epi16_mask(z4, z0, 1); + _mm512_mask_compressstoreu_epi16(WC_PW(rdi, 0), k1, z4); + rbx = (word32)((word32)k1); + rbx = (word32)(WC_X64I_POPCNT32((word32)rbx)); + rdi = (word64)(rdi + rbx * 2); + rsi = (word32)((word32)rsi - (word32)rbx); + rdx = (word64)(rdx + 0x30); + rcx = (word32)((word32)rcx - 0x30); + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_fast; +L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_fast_end: +L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_tail: + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_tail_end; + } + if ((sword32)((word32)rcx) < (sword32)(3)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_tail_end; + } + r8 = (word32)((word32)WC_L16(rdx, 0)); + r8 = (word32)((word32)r8 & 0xfff); + r9 = (word32)((word32)WC_L16(rdx, 1)); + r9 = (word32)((word32)r9 >> 4); + if ((sword32)((word32)r8) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_skip0; + } + WC_S16(rdi, 0) = (word16)((word16)r8); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_tail_end; + } +L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_skip0: + if ((sword32)((word32)r9) >= (sword32)(0xd01)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_skip1; + } + WC_S16(rdi, 0) = (word16)((word16)r9); + rdi = (word64)(rdi + 2); + rsi = (word32)((word32)rsi - 1); + if (((word32)rsi) == (0)) { + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_tail_end; + } +L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_skip1: + rdx = (word64)(rdx + 3); + rcx = (word32)((word32)rcx - 3); + goto L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_tail; +L_mlkem_rej_uniform_n_avx512_vbmi_vbmi2_tail_end: + rax = (word32)((word32)rax - (word32)rsi); + return (unsigned int)(word32)rax; +} + +#endif /* HAVE_INTEL_AVX512_VBMI2 */ +#endif /* HAVE_INTEL_AVX512_VBMI */ +#endif /* WOLFSSL_HAVE_MLKEM */ + +#undef WC_PR +#undef WC_PW +#undef WC_L8 +#undef WC_L16 +#undef WC_L32 +#undef WC_L64 +#undef WC_S8 +#undef WC_S16 +#undef WC_S32 +#undef WC_S64 +#undef WC_X64I_TARGET +#undef WC_X64I_UNUSED +#undef WC_X64I_MUL128 +#undef WC_X64I_POPCNT32 +#undef WC_X64I_POPCNT64 +#undef WC_X64I_BSR64 +#undef WC_X64I_BSWAP64 +#undef WC_X64I_DIV128 + +#endif /* WOLFSSL_X86_64_BUILD */